From b55815086a9eafae73cd79b19db7e09c557e2eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Tue, 19 Nov 2024 08:32:08 +0000 Subject: [PATCH] refactor travis tests for the new release cadence (#1192) Co-authored-by: Cas Donoghue --- .ci/docker-setup.sh | 111 +++++++++++++++++++++++++++----------------- .ci/logstash-run.sh | 2 +- .travis.yml | 24 ++++++---- 3 files changed, 84 insertions(+), 53 deletions(-) diff --git a/.ci/docker-setup.sh b/.ci/docker-setup.sh index 8e03b1c9..52283c52 100755 --- a/.ci/docker-setup.sh +++ b/.ci/docker-setup.sh @@ -1,61 +1,88 @@ #!/bin/bash -# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh` +# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` # Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. set -e -VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json" - - pull_docker_snapshot() { project="${1?project name required}" - local docker_image="docker.elastic.co/${project}/${project}:${ELASTIC_STACK_VERSION}" + local docker_image="docker.elastic.co/${project}/${project}${DISTRIBUTION_SUFFIX}:${ELASTIC_STACK_VERSION}" echo "Pulling $docker_image" - docker pull "$docker_image" + if docker pull "$docker_image" ; then + echo "docker pull successful" + else + case "$ELASTIC_STACK_VERSION_ARG" in + "8.previous"|"8.current"|"8.next") + exit 1 + ;; + *) + exit 2 + ;; + esac + fi } -if [ "$ELASTIC_STACK_VERSION" ]; then - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --silent $VERSION_URL) - if [[ "$SNAPSHOT" = "true" ]]; then - ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - echo $ELASTIC_STACK_RETRIEVED_VERSION - else - ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then - # remove starting and trailing double quotes - ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}" - ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}" - echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}" - export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION - fi +VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" + +if [ -z "${ELASTIC_STACK_VERSION}" ]; then + echo "Please set the ELASTIC_STACK_VERSION environment variable" + echo "For example: export ELASTIC_STACK_VERSION=7.x" + exit 1 +fi - echo "Testing against version: $ELASTIC_STACK_VERSION" +# save the original arg if needed +ELASTIC_STACK_VERSION_ARG="$ELASTIC_STACK_VERSION" - if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then - pull_docker_snapshot "logstash" - if [ "$INTEGRATION" == "true" ]; then - pull_docker_snapshot "elasticsearch" - fi - fi +echo "Fetching versions from $VERSION_URL" +VERSIONS=$(curl -s $VERSION_URL) - if [ -f Gemfile.lock ]; then - rm Gemfile.lock - fi +if [[ "$SNAPSHOT" = "true" ]]; then + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"') + echo $ELASTIC_STACK_RETRIEVED_VERSION +else + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"') +fi - cd .ci +if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then + # remove starting and trailing double quotes + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}" + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}" + echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}" + export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION +elif [[ "$ELASTIC_STACK_VERSION" == "8.next" ]]; then + # we know "8.next" only exists between FF and GA of a minor + # exit 1 so the build is skipped + exit 1 +fi + +case "${DISTRIBUTION}" in + default) DISTRIBUTION_SUFFIX="" ;; # empty string when explicit "default" is given + *) DISTRIBUTION_SUFFIX="${DISTRIBUTION/*/-}${DISTRIBUTION}" ;; +esac +export DISTRIBUTION_SUFFIX + +echo "Testing against version: $ELASTIC_STACK_VERSION (distribution: ${DISTRIBUTION:-"default"})" +if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then + pull_docker_snapshot "logstash" if [ "$INTEGRATION" == "true" ]; then - docker-compose down - docker-compose build --quiet - else - docker-compose down - docker-compose build logstash --quiet + pull_docker_snapshot "elasticsearch" fi -else - echo "Please set the ELASTIC_STACK_VERSION environment variable" - echo "For example: export ELASTIC_STACK_VERSION=6.2.4" - exit 1 fi +if [ -f Gemfile.lock ]; then + rm Gemfile.lock +fi + +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + +cd .ci + +export BUILDKIT_PROGRESS=plain +if [ "$INTEGRATION" == "true" ]; then + docker-compose down + docker-compose build --quiet +else + docker-compose down + docker-compose build logstash --quiet +fi diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 9f2d46d5..0801282b 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -22,7 +22,7 @@ wait_for_es() { [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo $(curl $CURL_OPTS -vi $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") + echo $(curl $CURL_OPTS -vi $ES_URL | jq -r .version.number) } if [[ "$INTEGRATION" != "true" ]]; then diff --git a/.travis.yml b/.travis.yml index 077d7e9e..769f52de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,17 @@ import: - logstash-plugins/.ci:travis/travis.yml@1.x -env: - - INTEGRATION=false ELASTIC_STACK_VERSION=7.x - - INTEGRATION=false ELASTIC_STACK_VERSION=8.x SNAPSHOT=true - - INTEGRATION=true ELASTIC_STACK_VERSION=7.x - - INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info - - INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info - - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info - - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info - - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info - - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info +jobs: + include: + - stage: "Integration Tests" + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future +# - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main + - stage: "Secure Integration Tests" + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_KEY_INVALID=true + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3