From 8cf004b77abcc226b3fa45e7dfd1f08fa18f12d4 Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Thu, 26 Sep 2024 16:02:56 -0400 Subject: [PATCH] CI lib support; general updates (#141) * ci-library-support: fr * Fix * Sync * ci-library-support: sync * ci-library-support: test * ci-library-support: test alpha * ci-library-support: fix path * ci-library-support: publish alpha * ci-library-support: switch to alpha * ci-library-support: drop an info message in reduce --- .circleci/config.yml | 5 +-- .circleci/library-config.yml | 46 ++++++++++++++++++++++++++++ .circleci/src.yml | 47 +---------------------------- scripts/circleci-config-validate.sh | 12 ++++++-- src/commands/filter.yml | 8 +++++ src/commands/reduce.yml | 8 +++++ src/jobs/continue.yml | 9 ++++++ src/scripts/filter.sh | 3 ++ src/scripts/reduce.sh | 20 ++++++++++++ 9 files changed, 108 insertions(+), 50 deletions(-) create mode 100644 .circleci/library-config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 5260dd1..7a04ee8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,8 +6,8 @@ setup: true orbs: orb-tools: circleci/orb-tools@11.6.1 circleci-cli: circleci/circleci-cli@0.1.9 - # dynamic: bjd2385/dynamic-continuation@dev:alpha - dynamic: bjd2385/dynamic-continuation@3.8.2 + dynamic: bjd2385/dynamic-continuation@dev:alpha + # dynamic: bjd2385/dynamic-continuation@3.8.2 general: premiscale/general@1.2.8 slack: circleci/slack@5.0.0 @@ -18,6 +18,7 @@ workflows: # Development branches - dynamic/continue: + library-config: library-config context: orb-publishing # On tag diff --git a/.circleci/library-config.yml b/.circleci/library-config.yml new file mode 100644 index 0000000..3f6716a --- /dev/null +++ b/.circleci/library-config.yml @@ -0,0 +1,46 @@ +jobs: + bats-tests: + parameters: + yq-version: + description: |+ + Version of yq to install. + + https://github.com/mikefarah/yq/releases + type: string + default: 4.40.5 + tests-directory: + description: Directory in which bats tests reside. + type: string + default: tests/ + formatter: + description: Bats formatter. + type: string + default: tap + resource-class: + description: Resource class to execute as. + type: enum + default: small + enum: + - small + - medium + - large + - 2xlarge + executor: default + resource_class: << parameters.resource-class >> + steps: + - checkout + - run: + name: Install bats + command: |+ + sudo apt update + sudo apt install -y bats + - run: + name: Install yq + command: |+ + wget https://github.com/mikefarah/yq/releases/download/v<< parameters.yq-version >>/yq_linux_amd64 -o yq + sudo install yq /usr/bin/yq + rm yq + - run: + name: Bats tests + command: |+ + bats --formatter << parameters.formatter >> --timing --recursive << parameters.tests-directory >> \ No newline at end of file diff --git a/.circleci/src.yml b/.circleci/src.yml index 2072612..da471db 100644 --- a/.circleci/src.yml +++ b/.circleci/src.yml @@ -14,52 +14,7 @@ executors: - image: cimg/base:2024.09 -jobs: - bats-tests: - parameters: - yq-version: - description: |+ - Version of yq to install. - - https://github.com/mikefarah/yq/releases - type: string - default: 4.40.5 - tests-directory: - description: Directory in which bats tests reside. - type: string - default: tests/ - formatter: - description: Bats formatter. - type: string - default: tap - resource-class: - description: Resource class to execute as. - type: enum - default: small - enum: - - small - - medium - - large - - 2xlarge - executor: default - resource_class: << parameters.resource-class >> - steps: - - checkout - - run: - name: Install bats - command: |+ - sudo apt update - sudo apt install -y bats - - run: - name: Install yq - command: |+ - wget https://github.com/mikefarah/yq/releases/download/v<< parameters.yq-version >>/yq_linux_amd64 -o yq - sudo install yq /usr/bin/yq - rm yq - - run: - name: Bats tests - command: |+ - bats --formatter << parameters.formatter >> --timing --recursive << parameters.tests-directory >> +jobs: {} workflows: diff --git a/scripts/circleci-config-validate.sh b/scripts/circleci-config-validate.sh index 7ef8242..cf60cc7 100755 --- a/scripts/circleci-config-validate.sh +++ b/scripts/circleci-config-validate.sh @@ -8,12 +8,20 @@ if [[ -n $CIRCLECI ]]; then fi if ! command -v circleci &>/dev/null; then - echo "Circleci CLI could not be found. Install the latest CLI version https://circleci.com/docs/2.0/local-cli/#installation" + echo "Circleci CLI could not be found. Install the latest CLI version: https://circleci.com/docs/2.0/local-cli/#installation" exit 1 fi +if ! command -v yq &>/dev/null; then + echo "yq could not be found. Install the latest yq version: https://github.com/mikefarah/yq/releases" + exit 1 +fi + + +lib="$(grep -oP "(?<=library-config: ).*" .circleci/config.yml)" + for config in "$@"; do - if ! reMSG=$(circleci config validate --skip-update-check -c "$config"); then + if ! reMSG=$( circleci config validate --skip-update-check -c <(yq -Mr eval-all "explode(.) as \$item ireduce ( {}; . * \$item )" <(printf "%s\\n%s" "$config" "$lib" ) ) ); then printf "CircleCI config file \"%s\" failed validation.\\n" "$config" echo "${reMSG}" exit 1 diff --git a/src/commands/filter.yml b/src/commands/filter.yml index e0c5faa..0c08c39 100644 --- a/src/commands/filter.yml +++ b/src/commands/filter.yml @@ -37,6 +37,13 @@ parameters: description: Provides the ability to map root repository changes (./) to a config file name. type: string default: app + library-config: + description: |+ + Specify a common configuration file that serves as a library of commands and jobs. + + This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution. + type: string + default: cilib reporting-window: description: The time window used to calculate summary metrics for the default branch of the repository. Defaults to disabled. Allows users to force all workflows to run if no workflows have been ran in the time window. type: enum @@ -83,6 +90,7 @@ steps: SH_DYNAMIC_CONTINUATION_DEBUG: << parameters.debug >> SH_FORCE_ALL: << parameters.force-all >> SH_INCLUDE_CONFIG_CHANGES: << parameters.include-config-changes >> + SH_LIBRARY_CONFIG: << parameters.library-config >> SH_MODULES: << parameters.modules >> SH_MODULES_FILTERED: << parameters.modules-filtered >> SH_PROJECT_TYPE: << parameters.project-type >> diff --git a/src/commands/reduce.yml b/src/commands/reduce.yml index 3996eb1..4f272f2 100644 --- a/src/commands/reduce.yml +++ b/src/commands/reduce.yml @@ -14,6 +14,13 @@ parameters: description: Provides the ability to map root repository changes (./) to a config file name. Defaults to app.yml. type: string default: app + library-config: + description: |+ + Specify a common configuration file that serves as a library of commands and jobs. + + This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution. + type: string + default: '' continue-config: description: Path to the internally-used config for continuation type: string @@ -49,6 +56,7 @@ steps: environment: SH_CONTINUE_CONFIG: << parameters.continue-config >> SH_DYNAMIC_CONTINUATION_DEBUG: << parameters.debug >> + SH_LIBRARY_CONFIG: << parameters.library-config >> SH_MODULES_FILTERED: << parameters.modules-filtered >> SH_ROOT_CONFIG: << parameters.root-config >> command: << include(scripts/reduce.sh) >> \ No newline at end of file diff --git a/src/jobs/continue.yml b/src/jobs/continue.yml index 0a8b052..c00b611 100644 --- a/src/jobs/continue.yml +++ b/src/jobs/continue.yml @@ -74,6 +74,13 @@ parameters: default: .circleci/continue-config.yml # both + library-config: + description: |+ + Specify a common configuration file that serves as a library of commands and jobs. + + This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution. + type: string + default: '' root-config: description: Provides the ability to map root repository changes (./) to a config file name. Name should be left without extension. type: string @@ -164,6 +171,7 @@ steps: force-all: << parameters.force-all >> base-revision: << parameters.base-revision >> root-config: << parameters.root-config >> + library-config: << parameters.library-config >> reporting-window: << parameters.reporting-window >> circle-organization: << parameters.circle-organization >> circle-token: << parameters.circle-token >> @@ -178,6 +186,7 @@ steps: modules-filtered: << parameters.modules-filtered >> continue-config: << parameters.continue-config >> root-config: << parameters.root-config >> + library-config: << parameters.library-config >> circle-token: << parameters.circle-token >> project-type: << parameters.project-type >> cache: false diff --git a/src/scripts/filter.sh b/src/scripts/filter.sh index eb01438..3be6876 100755 --- a/src/scripts/filter.sh +++ b/src/scripts/filter.sh @@ -71,6 +71,7 @@ while [ "${SH_CIRCLE_ORGANIZATION:0:1}" = '$' ]; do SH_CIRCLE_ORGANIZATION="$(eval echo "$SH_CIRCLE_ORGANIZATION")" done + # CircleCI API token should be set. if [ -z "$SH_CIRCLE_TOKEN" ]; then error "must set CircleCI token for successful authentication." @@ -81,10 +82,12 @@ fi # Move yaml files -> yml so we can handle both extensions for YAML configs. Not that we want both, but we should handle both cases. for f in .circleci/*.yaml; do warn "migrating pipeline \"$f\" -> \"${f%.*}.yml\"" + if [ -f "${f%.*}.yml" ]; then error "could not migrate \"$f\", \"${f%.*}.yml\" already exists." exit 1 fi + mv "$f" "${f%.*}.yml" done diff --git a/src/scripts/reduce.sh b/src/scripts/reduce.sh index 0f65abc..5d24a45 100755 --- a/src/scripts/reduce.sh +++ b/src/scripts/reduce.sh @@ -1,6 +1,20 @@ # shellcheck disable=SC2288,SC2001,SC2148,SC2002,SC2016,SC2046 +info() +{ + if [ $# -ne 1 ]; then + printf "Function \"info\" expected at least 1 argument: info message.\\n" >&2 + exit 1 + fi + + local msg + msg="$1" + + printf "INFO: %s\\n" "$msg" +} + + # If `modules` is unavailable, stop this job without continuation if [ ! -f "$SH_MODULES_FILTERED" ] || [ ! -s "$SH_MODULES_FILTERED" ]; then printf "Nothing to merge. Halting the job.\\n" @@ -19,4 +33,10 @@ awk "{ }" "$SH_MODULES_FILTERED" > /tmp/"$CIRCLE_WORKFLOW_ID.txt" mv /tmp/"$CIRCLE_WORKFLOW_ID.txt" "$SH_MODULES_FILTERED" +# Append the library config, if it is specified and exists, to the reduction. +if [ "$SH_LIBRARY_CONFIG" != "" ] && [ -f .circleci/"$SH_LIBRARY_CONFIG".yml ]; then + info "Merging user-specified library config \"%s\" (.circleci/%s.yml) with final config.\\n" "$SH_LIBRARY_CONFIG" + printf ".circleci/%s.yml" "$SH_LIBRARY_CONFIG" >> "$SH_MODULES_FILTERED" +fi + yq -Mr eval-all 'explode(.) as $item ireduce ( {}; . * $item )' $(cat "$SH_MODULES_FILTERED" | xargs) | tee "$SH_CONTINUE_CONFIG" \ No newline at end of file