diff --git a/logstash-filter-xml.yaml b/logstash-filter-xml.yaml new file mode 100644 index 00000000000..f2de4a5a0d5 --- /dev/null +++ b/logstash-filter-xml.yaml @@ -0,0 +1,68 @@ +# +# WARNING: Due to the way logstash loads plugins we have to load this package +# at build time. That means the logstash package must be rebuilt to +# pick up changes from this package. Simply building and publishing +# a new version of this package is not enough to land changes. +# +# tl;dr: If you're touching this package please also rebuild logstash!! +# +package: + name: logstash-filter-xml + version: 4.2.0 + epoch: 0 + description: Logstash - transport and process your logs, events, or other data + copyright: + - license: Apache-2.0 + +environment: + contents: + packages: + - bash + - busybox + - ca-certificates-bundle + - jruby-9.4 + - jruby-9.4-default-ruby + - openjdk-17 + - openjdk-17-default-jvm + environment: + JAVA_HOME: /usr/lib/jvm/default-jvm + +vars: + gem: logstash-filter-xml + +# Do not run ruby/clean as logstash needs the cached gem file to install +pipeline: + - uses: git-checkout + with: + repository: https://github.com/logstash-plugins/logstash-filter-xml.git + tag: v${{package.version}} + expected-commit: e947ec5b08e8b25ee35aba1bb0bcd070b1454dae + - runs: | + # Fix GHSA-xc9x-jj77-9p9j + sed -i "s/'nokogiri', '>= 1.13.8'/'nokogiri', '>= 1.16.2'/" "${{package.name}}.gemspec" + - uses: ruby/build + with: + gem: ${{vars.gem}} + - uses: ruby/install + with: + gem: ${{vars.gem}} + version: ${{package.version}} + +test: + environment: + contents: + packages: + - wolfi-base + - logstash + - openjdk-17-default-jvm + pipeline: + - runs: | + logstash-plugin install $(find / -type f -name "${{package.name}}-*.gem") + logstash-plugin list | grep ${{package.name}} + +update: + enabled: true + github: + identifier: logstash-plugins/logstash-filter-xml + use-tag: true + strip-prefix: v diff --git a/logstash-integration-jdbc.yaml b/logstash-integration-jdbc.yaml index accc03da0f8..903578c3b51 100644 --- a/logstash-integration-jdbc.yaml +++ b/logstash-integration-jdbc.yaml @@ -1,3 +1,11 @@ +# +# WARNING: Due to the way logstash loads plugins we have to load this package +# at build time. That means the logstash package must be rebuilt to +# pick up changes from this package. Simply building and publishing +# a new version of this package is not enough to land changes. +# +# tl;dr: If you're touching this package please also rebuild logstash!! +# package: name: logstash-integration-jdbc version: 5.4.8 @@ -12,44 +20,47 @@ environment: - build-base - busybox - ca-certificates-bundle - - openjdk-11 - - openjdk-11-default-jvm - - ruby-3.2 + - jruby-9.4 + - jruby-9.4-default-ruby + - openjdk-17 + - openjdk-17-default-jvm +# Do not run ruby/clean as logstash needs the cached gem file to install pipeline: - uses: git-checkout with: repository: https://github.com/logstash-plugins/logstash-integration-jdbc tag: v${{package.version}} expected-commit: 68ec2357b7007d753ba0e77f253e6c5941854889 - - # update the derby version to mitigate the CVE-2018-1313 - - runs: sed -i "s/def DERBY_VERSION = '10.14.1.0'/def DERBY_VERSION = '10.14.2.0'/g" build.gradle - - runs: | - # Extract the Ruby version - ruby_version=$(ruby -e 'puts RUBY_VERSION') - dest="${{targets.destdir}}/usr/share/java/logstash/vendor/bundle/jruby/${ruby_version}/gems/${{vars.gem}}-${{package.version}}" - mkdir -p "$dest" + # update the derby version to mitigate the CVE-2018-1313 + sed -i "s/def DERBY_VERSION = '10.14.1.0'/def DERBY_VERSION = '10.14.2.0'/g" build.gradle ./gradlew vendor - mv vendor "$dest" - - uses: ruby/build with: gem: ${{vars.gem}} - - uses: ruby/install with: gem: ${{vars.gem}} - gem-file: ${{vars.gem}}.gemspec version: ${{package.version}} - - uses: ruby/clean - vars: gem: logstash-integration-jdbc +test: + environment: + contents: + packages: + - wolfi-base + - logstash + - openjdk-11-default-jvm + - jruby-9.4 + pipeline: + - runs: | + logstash-plugin install $(find / -type f -name "${{package.name}}-*.gem") + logstash-plugin list | grep ${{package.name}} + update: enabled: true github: diff --git a/logstash-output-opensearch.yaml b/logstash-output-opensearch.yaml new file mode 100644 index 00000000000..369d5147998 --- /dev/null +++ b/logstash-output-opensearch.yaml @@ -0,0 +1,67 @@ +# +# WARNING: Due to the way logstash loads plugins we have to load this package +# at build time. That means the logstash package must be rebuilt to +# pick up changes from this package. Simply building and publishing +# a new version of this package is not enough to land changes. +# +# tl;dr: If you're touching this package please also rebuild logstash!! +# +package: + name: logstash-output-opensearch + version: 2.0.2 + epoch: 0 + description: Logstash - transport and process your logs, events, or other data + copyright: + - license: Apache-2.0 + +environment: + contents: + packages: + - bash + - busybox + - ca-certificates-bundle + - jruby-9.4 + - jruby-9.4-default-ruby + - openjdk-17 + - openjdk-17-default-jvm + environment: + JAVA_HOME: /usr/lib/jvm/default-jvm + +vars: + gem: logstash-output-opensearch + +# Do not run ruby/clean as logstash needs the cached gem file to install +pipeline: + - uses: git-checkout + with: + repository: https://github.com/opensearch-project/logstash-output-opensearch + tag: ${{package.version}} + expected-commit: 844fb28d9e148e4042563d277b8e4c8a63392f81 + - uses: ruby/build + with: + gem: ${{vars.gem}} + - runs: | + # Get rid of the `-VERSION-ARCH-OS` suffix from the gem file name. + mv ${{vars.gem}}-*.gem ${{vars.gem}}-${{package.version}}.gem + - uses: ruby/install + with: + gem: ${{vars.gem}} + version: ${{package.version}} + +test: + environment: + contents: + packages: + - wolfi-base + - logstash + - openjdk-17-default-jvm + pipeline: + - runs: | + logstash-plugin install $(find / -type f -name "${{package.name}}-*.gem") + logstash-plugin list | grep ${{package.name}} + +update: + enabled: true + github: + identifier: opensearch-project/logstash-output-opensearch + use-tag: true diff --git a/logstash.yaml b/logstash.yaml index 0decc00950c..00ebb11fa82 100644 --- a/logstash.yaml +++ b/logstash.yaml @@ -1,15 +1,32 @@ +# Logstash is a bit of a behemoth because of the way the plugin system works. +# Usually, patched plugins would be dropped on the filesystem as APK's and the +# application could just pick them up. Logstash's plugin system requires the +# plugins to be added by the logstash-plugin cli tool since it uses a generated +# Gemfile. +# +# In order to patch plugins we can do a few things +# +# 1. If the plugin is a default plugin, patch it out of the plugins-metadata.json +# file so the upstream plugin is not installed. Add the plugin apk to the build +# environment and use logstash-plugin to install the .gem file before packaging +# the dist into a tarball (and eventually apk). All of this should be done in +# the main pipeline so the changes are picked up by all subpackages. +# +# 2. If the plugin is not a default plugin, do the same as above, but in a +# subpackage to be installed separately. package: name: logstash version: 8.12.1 - epoch: 0 + epoch: 1 description: Logstash - transport and process your logs, events, or other data copyright: - license: Apache-2.0 dependencies: + provides: + - ${{package.name}}-base=${{package.full-version}} runtime: - bash # some helper scripts use bash - - logstash-integration-jdbc - - openjdk-11-jre + - openjdk-17-jre environment: contents: @@ -19,10 +36,25 @@ environment: - ca-certificates-bundle - curl - gradle + - jq + - jruby-9.4 + # logstash plugins to load at build-time + - logstash-filter-xml - logstash-integration-jdbc - - openjdk-11 - - openjdk-11-default-jvm - - ruby-3.2 + - logstash-output-opensearch + # end logstash plugins to load at build-time + - openjdk-17 + - openjdk-17-default-jvm + environment: + OSS: "true" + LOGSTASH_SOURCE: "1" + LANG: en_US.UTF-8 + JAVA_HOME: /usr/lib/jvm/default-jvm + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + LS_DESTINATION: usr/share/logstash + MAVEN_OPTS: "-Dmaven.repo.local=/var/cache/melange" + GRADLE_USER_HOME: "/var/cache/melange" + GRADLE_OPTS: "-Dorg.gradle.caching=true" pipeline: - uses: git-checkout @@ -31,46 +63,85 @@ pipeline: tag: v${{package.version}} expected-commit: c7e185a56af665671d38ad8dbf37bb5817f50678 - - runs: | - # disable the logstash-integration-jdbc plugin download as we build and package it separately - sed -i '/"logstash-integration-jdbc":/,/},/ s/"default-plugins": true/"default-plugins": false/' rakelib/plugins-metadata.json + - name: Patch sources + runs: | + # Disable the logstash-integration-jdbc plugin download as we build and + # package it separately + jq 'del(.["logstash-integration-jdbc"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json + mv /tmp/plugins-metadata.json rakelib/ - - runs: | - export OSS=true - export LOGSTASH_SOURCE=1 - export LANG=en_US.UTF-8 - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk - export LS_JAVA_HOME=/usr/lib/jvm/java-11-openjdk + # Disable the logstash-filter-xml plugin download as we build and + # package it separately + jq 'del(.["logstash-filter-xml"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json + mv /tmp/plugins-metadata.json rakelib/ - # Disable -Werror which means that warnings are not treated as errors during compilation. Required for Java 21. + # Disable -Werror which means that warnings are not treated as errors + # during compilation. Required for Java 21. sed -i '/options.compilerArgs.add("-Werror")/s/^/\/\//' build.gradle - gem install rake - gem install bundler - - # A hack-fix to tackle with `Could not find method enabled() for arguments [true] on Report xml` error starting after 8.11.x. + # A hack-fix to tackle with `Could not find method enabled() for + # arguments [true] on Report xml` error starting after 8.11.x. sed -i '/enabled/d' logstash-core/build.gradle - gradle installDevelopmentGems - gradle installDefaultGems - rake bootstrap - rake plugin:install-default + - name: Patch out the windows and mac tarball creation + uses: patch + with: + patches: create_archive_pack.patch + + - name: Install all the default plugins + runs: | + ./gradlew installDefaultGems + + - name: Re-inject default plugins that have been patched + runs: | + bin/logstash-plugin install \ + $(find / -type f -name "logstash-integration-jdbc-*.gem") \ + $(find / -type f -name "logstash-filter-xml-*.gem") - gradle assembleOssTarDistribution -Pjdk_bundle_os=linux + - name: Build tarball distribution + runs: | + ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux - mkdir -p ${{targets.destdir}}/usr/share/java/logstash - tar --strip-components 1 -C ${{targets.destdir}}/usr/share/java/logstash -xf build/logstash-oss-*-SNAPSHOT-no-jdk.tar.gz + - name: Package distribution + runs: | + mkdir -p ${{targets.contextdir}}/${LS_DESTINATION} + tar --strip-components 1 \ + -C ${{targets.contextdir}}/${LS_DESTINATION} \ + -xf $(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz") - mkdir -p ${{targets.destdir}}/usr/bin/$name - for i in ${{targets.destdir}}/usr/share/java/logstash/bin/*; do + mkdir -p ${{targets.contextdir}}/usr/bin/$name + for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do name=$(basename $i) - ln -sf /usr/share/java/logstash/bin/$name ${{targets.destdir}}/usr/bin/$name + ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name done + - name: Patch CVEs + runs: | # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855 - rm -rf ${{targets.destdir}}/usr/share/java/logstash/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib + rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib subpackages: + - name: ${{package.name}}-compat + description: Compatibility with the upstream image + pipeline: + - runs: | + src=docker/data/logstash + dst="${{targets.contextdir}}/${LS_DESTINATION}" + mkdir -p "${dst}/config" \ + "${dst}/pipeline" \ + "${{targets.contextdir}}/usr/bin" + + install -m644 "${src}/config/pipelines.yml" "${dst}/config/pipelines.yml" + install -m644 "${src}/config/logstash-oss.yml" "${dst}/config/logstash-oss.yml" + install -m644 "${src}/config/logstash-full.yml" "${dst}/config/logstash-full.yml" + install -m644 "${src}/config/log4j2.file.properties" "${dst}/config/log4j2.file.properties" + install -m644 "${src}/config/log4j2.properties" "${dst}/config/log4j2.properties" + install -m644 "${src}/pipeline/default.conf" "${dst}/pipeline/default.conf" + install -m755 "${src}/bin/docker-entrypoint" "${{targets.contextdir}}/usr/bin/docker-entrypoint" + dependencies: + runtime: + - ${{package.name}}-base + - name: ${{package.name}}-env2yaml description: Merge environment variables into logstash.yml pipeline: @@ -87,6 +158,80 @@ subpackages: subpackage: "true" - uses: strip + # Due to the way logstash implements their plugin system, this is a full + # logstash package with the opensearch plugin bundled. It is based off the + # same artifacts in the main package, meaning it will pick up the changes to + # the default gems that were made in the main package it just adds the + # opensearch plugin. + - name: ${{package.name}}-with-output-opensearch + description: Build logstash with logstash-output-opensearch plugin + pipeline: + - name: Re-inject default plugins that have been patched + runs: | + bin/logstash-plugin install \ + $(find / -type f -name "logstash-output-opensearch-*.gem") + - name: Build tarball distribution + runs: | + ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux + - name: Package distribution + runs: | + mkdir -p ${{targets.contextdir}}/${LS_DESTINATION} + tar --strip-components 1 \ + -C ${{targets.contextdir}}/${LS_DESTINATION} \ + -xf $(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz") + + mkdir -p ${{targets.contextdir}}/usr/bin/$name + for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do + name=$(basename $i) + ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name + done + - name: Patch CVEs + runs: | + # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855 + rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib + dependencies: + provides: + - ${{package.name}}-base=${{package.full-version}} + test: + environment: + contents: + packages: + - bash + - wolfi-base + - openjdk-17-jre + - openjdk-17-default-jvm + environment: + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + pipeline: + - name: Ensure output opensearch plugin was installed + runs: | + logstash-plugin list | grep logstash-output-opensearch + - name: Ensure logstash itself works with basic function + runs: | + echo "hello" | \ + logstash -e 'input { stdin { } } output { stdout {} }' | \ + grep 'message.*hello' + +test: + environment: + contents: + packages: + - wolfi-base + - openjdk-17-jre + - openjdk-17-default-jvm + environment: + LS_JAVA_HOME: /usr/lib/jvm/default-jvm + pipeline: + - name: Ensure default plugins were actually installed + runs: | + logstash-plugin list | grep logstash-integration-jdbc + logstash-plugin list | grep logstash-filter-xml + - name: Ensure logstash itself works with basic function + runs: | + echo "hello" | \ + logstash -e 'input { stdin { } } output { stdout {} }' | \ + grep 'message.*hello' + update: enabled: true github: diff --git a/logstash/create_archive_pack.patch b/logstash/create_archive_pack.patch new file mode 100644 index 00000000000..b86a96d9716 --- /dev/null +++ b/logstash/create_archive_pack.patch @@ -0,0 +1,23 @@ +diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake +index 88e765e93..7524bbd4c 100644 +--- a/rakelib/artifacts.rake ++++ b/rakelib/artifacts.rake +@@ -194,17 +194,11 @@ namespace "artifact" do + + desc "Build all (jdk bundled and not) OSS tar.gz and zip of default logstash plugins with all dependencies" + task "archives_oss" => ["prepare", "generate_build_metadata"] do +- #with bundled JDKs +- @bundles_jdk = true +- license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths] +- create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin") +- create_archive_pack(license_details, "arm64", "linux", "darwin") +- + #without JDK + @bundles_jdk = false ++ license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths] + safe_system("./gradlew bootstrap") #force the build of Logstash jars + build_tar(*license_details, platform: '-no-jdk') +- build_zip(*license_details, platform: '-no-jdk') + end + + desc "Build an RPM of logstash with all dependencies" diff --git a/ruby3.2-logstash-output-opensearch.yaml b/ruby3.2-logstash-output-opensearch.yaml deleted file mode 100644 index 444addc7810..00000000000 --- a/ruby3.2-logstash-output-opensearch.yaml +++ /dev/null @@ -1,61 +0,0 @@ -package: - name: ruby3.2-logstash-output-opensearch - version: 2.0.2 - epoch: 1 - description: Logstash - transport and process your logs, events, or other data - copyright: - - license: Apache-2.0 - dependencies: - runtime: - - ruby3.2-aws-sdk-core - - ruby3.2-json - - ruby3.2-logstash-core-plugin-api - - ruby3.2-manticore - - ruby3.2-stud - -environment: - contents: - packages: - - bash - - busybox - - ca-certificates-bundle - - jruby-9.4 - - jruby-9.4-default-ruby - - openjdk-11 - - openjdk-11-default-jvm - environment: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk - -vars: - gem: logstash-output-opensearch - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/opensearch-project/logstash-output-opensearch - tag: ${{package.version}} - expected-commit: 844fb28d9e148e4042563d277b8e4c8a63392f81 - - - runs: | - mkdir -p "${{targets.destdir}}"/usr/share/logstash/config - cp release/docker/logstash-opensearch-sample.conf "${{targets.destdir}}"/usr/share/logstash/config/ - - - runs: gem install bundler - - - runs: jruby -S gem build ${{vars.gem}}.gemspec - - # Get rid of the `-VERSION-ARCH-OS` suffix from the gem file name. - - runs: mv ${{vars.gem}}-*.gem ${{vars.gem}}.gem - - - uses: ruby/install - with: - gem-file: ${{vars.gem}}.gem - version: ${{package.version}} - - - uses: ruby/clean - -update: - enabled: true - github: - identifier: opensearch-project/logstash-output-opensearch - use-tag: true