From ff8e05d502211adc50dce7dd538fa707f4339d82 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Mon, 14 Sep 2020 14:53:34 -0400 Subject: [PATCH 1/2] Create ceph-pr-combined job This will run `make check` and then the Ceph API tests against the built binaries instead of us running `make` twice per PR then running API tests against one of those runs. Signed-off-by: David Galloway --- ceph-pr-combined/build/api/start | 7 + ceph-pr-combined/build/api/success | 7 + ceph-pr-combined/build/failure | 14 ++ ceph-pr-combined/build/make-check/build | 18 +++ ceph-pr-combined/build/setup | 19 +++ .../config/definitions/ceph-pr-combined.yml | 135 ++++++++++++++++++ 6 files changed, 200 insertions(+) create mode 100644 ceph-pr-combined/build/api/start create mode 100644 ceph-pr-combined/build/api/success create mode 100644 ceph-pr-combined/build/failure create mode 100644 ceph-pr-combined/build/make-check/build create mode 100644 ceph-pr-combined/build/setup create mode 100644 ceph-pr-combined/config/definitions/ceph-pr-combined.yml diff --git a/ceph-pr-combined/build/api/start b/ceph-pr-combined/build/api/start new file mode 100644 index 000000000..b6bb32690 --- /dev/null +++ b/ceph-pr-combined/build/api/start @@ -0,0 +1,7 @@ +#!/bin/bash +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "github-status>0.0.3" ) +install_python_packages "pkgs[@]" +GITHUB_STATUS_STATE="pending" $VENV/github-status create diff --git a/ceph-pr-combined/build/api/success b/ceph-pr-combined/build/api/success new file mode 100644 index 000000000..80d6c391d --- /dev/null +++ b/ceph-pr-combined/build/api/success @@ -0,0 +1,7 @@ +#!/bin/bash +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "github-status>0.0.3" ) +install_python_packages "pkgs[@]" +GITHUB_STATUS_STATE="success" $VENV/github-status create diff --git a/ceph-pr-combined/build/failure b/ceph-pr-combined/build/failure new file mode 100644 index 000000000..62efb8feb --- /dev/null +++ b/ceph-pr-combined/build/failure @@ -0,0 +1,14 @@ +#!/bin/bash +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "github-status>0.0.3" ) +install_python_packages "pkgs[@]" + +# The end of the build/make-check/build writes this sentinel file if `make check` passes. +if [[ -f "$WORKSPACE/.makecheckpassed" ]]; then + GITHUB_STATUS_CONTEXT="ceph PR - API tests" +else + GITHUB_STATUS_CONTEXT="ceph PR - make check" +fi +GITHUB_STATUS_STATE="failure" $VENV/github-status create diff --git a/ceph-pr-combined/build/make-check/build b/ceph-pr-combined/build/make-check/build new file mode 100644 index 000000000..1b390c3fb --- /dev/null +++ b/ceph-pr-combined/build/make-check/build @@ -0,0 +1,18 @@ +#!/bin/bash +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "github-status>0.0.3" ) +install_python_packages "pkgs[@]" +GITHUB_STATUS_STATE="pending" $VENV/github-status create + +export NPROC=$(nproc) +export WITH_SEASTAR=true +timeout 3h ./run-make-check.sh +sleep 5 +ps -ef | grep -v jnlp | grep ceph || true + +# This is so ugly but in order for github-status to know if `make check` or API tests failed in the postbuildscript, the ceph-pr-combined/build/failure script can check for this sentinel file. +touch $WORKSPACE/.makecheckpassed + +GITHUB_STATUS_STATE="success" $VENV/github-status create diff --git a/ceph-pr-combined/build/setup b/ceph-pr-combined/build/setup new file mode 100644 index 000000000..6e0638960 --- /dev/null +++ b/ceph-pr-combined/build/setup @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +# set up variables needed for +# githubstatus to report back to the github PR +# if this project was started manually +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "github-status>0.0.3" ) +install_python_packages "pkgs[@]" + +# The github-status project checks if a job was triggered by github-pull-request-builder and will not updated PR statuses if it was. +# This is because github-pull-request-builder already updates PR statuses. However, we can't set stages in the status message. +# So I'm unsetting the ghprb vars to trick github-status into running. +# https://github.com/alfredodeza/github-status/blob/master/github_status/util.py +for var in ghprbActualCommit ghprbTriggerAuthor ghprbTargetBranch ghprbTriggerAuthorLogin ghprbCredentialsId ghprbGhRepository; do + unset $var +done diff --git a/ceph-pr-combined/config/definitions/ceph-pr-combined.yml b/ceph-pr-combined/config/definitions/ceph-pr-combined.yml new file mode 100644 index 000000000..aea739224 --- /dev/null +++ b/ceph-pr-combined/config/definitions/ceph-pr-combined.yml @@ -0,0 +1,135 @@ +- job: + name: ceph-pr-combined + description: 'make check and ceph API tests combined' + project-type: freestyle + defaults: global + concurrent: true + node: huge && bionic && x86_64 && !xenial && !trusty + display-name: 'ceph PRs: make check and API tests' + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + properties: + - build-discarder: + days-to-keep: 15 + num-to-keep: 300 + artifact-days-to-keep: -1 + artifact-num-to-keep: -1 + - github: + url: https://github.com/ceph/ceph/ + - rebuild: + auto-rebuild: true + - inject: + properties-content: | + TERM=xterm + + parameters: + - string: + name: ghprbPullId + description: "the GitHub pull id, like '72' in 'ceph/pull/72'" + + triggers: + - github-pull-request: + cancel-builds-on-update: true + allow-whitelist-orgs-as-admins: true + org-list: + - ceph + white-list-target-branches: + - master + - octopus + - nautilus + trigger-phrase: '^jenkins test make check|^jenkins test pr.*|^jenkins retest.*|^jenkins test api' + skip-build-phrase: '^jenkins do not test.*' + only-trigger-phrase: false + github-hooks: true + permit-all: true + auto-close-on-fail: false + # The option below requires jenkins-job-builder>=3.5.0 + no-commit-status: true + # I had hoped to get these working by running 'make check' and the API tests in the same job + # and somehow changing {stage} throughout the job but that doesn't appear to be possible. + # So I'm using github.com/alfredodeza/github-status +# status-context: "ceph PR: {stage}" +# started-status: "ceph PR: {stage} running" +# success-status: "ceph PR: {stage} succeeded" +# failure-status: "ceph PR: {stage} failed" + + scm: + - git: + url: https://github.com/ceph/ceph.git + branches: + - ${sha1} + refspec: +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/* + browser: auto + timeout: 20 + skip-tag: true + shallow-clone: true + wipe-workspace: true + + ## TODO: (maybe) I'm sure there's a cleaner/prettier way to do this. Maybe doing an actual Jenkins pipeline. + builders: + - inject: + properties-content: | + GITHUB_REPOSITORY="ceph/ceph" + GITHUB_STATUS_CONTEXT="ceph PR - make check" + GITHUB_STATUS_STARTED="running" + GITHUB_STATUS_SUCCESS="OK" + GITHUB_STATUS_FAILURE="failed" + GITHUB_STATUS_ERROR="completed with errors" + GITHUB_SHA=${ghprbActualCommit} + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/make-check/build + - inject: + properties-content: | + GITHUB_REPOSITORY="ceph/ceph" + GITHUB_STATUS_CONTEXT="ceph PR - API tests" + GITHUB_STATUS_STARTED="running" + GITHUB_STATUS_SUCCESS="OK" + GITHUB_STATUS_FAILURE="failed" + GITHUB_STATUS_ERROR="completed with errors" + GITHUB_SHA=${ghprbActualCommit} + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/api/start + - ../../../scripts/dashboard/install-backend-api-test-deps.sh + - shell: "cd src/pybind/mgr/dashboard; timeout 7200 ./run-backend-api-tests.sh" + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/api/success + + publishers: + - postbuildscript: + builders: + - role: SLAVE + build-on: + - FAILURE + - ABORTED + build-steps: + - inject: + properties-content: | + GITHUB_REPOSITORY="ceph/ceph" + GITHUB_STATUS_STARTED="running" + GITHUB_STATUS_SUCCESS="OK" + GITHUB_STATUS_FAILURE="failed" + GITHUB_STATUS_ERROR="completed with errors" + GITHUB_SHA=${ghprbActualCommit} + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/failure + + wrappers: + - ansicolor + - credentials-binding: + - text: + credential-id: 8dff73ff-506c-4397-969d-023f15aedea9 + variable: GITHUB_OAUTH_TOKEN From fc4121adaadb82cae477b4bebc665d8626bed062 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Mon, 14 Sep 2020 14:54:19 -0400 Subject: [PATCH 2/2] Disable ceph-pr-api and ceph-pull-requests individual jobs Replaced by ceph-pr-combined Signed-off-by: David Galloway --- ceph-pr-api/config/definitions/ceph-pr-api.yml | 1 + ceph-pull-requests/config/definitions/ceph-pull-requests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/ceph-pr-api/config/definitions/ceph-pr-api.yml b/ceph-pr-api/config/definitions/ceph-pr-api.yml index a9fa5b357..7bdacea32 100644 --- a/ceph-pr-api/config/definitions/ceph-pr-api.yml +++ b/ceph-pr-api/config/definitions/ceph-pr-api.yml @@ -1,4 +1,5 @@ - job: + disabled: true name: ceph-api project-type: freestyle defaults: global diff --git a/ceph-pull-requests/config/definitions/ceph-pull-requests.yml b/ceph-pull-requests/config/definitions/ceph-pull-requests.yml index 1ef21979b..0ed262697 100644 --- a/ceph-pull-requests/config/definitions/ceph-pull-requests.yml +++ b/ceph-pull-requests/config/definitions/ceph-pull-requests.yml @@ -1,4 +1,5 @@ - job: + disabled: true name: ceph-pull-requests project-type: freestyle defaults: global