CI lib support; general updates (#141)

* ci-library-support: fr * Fix * Sync * ci-library-support: sync * ci-library-support: test * ci-library-support: test alpha * ci-library-support: fix path * ci-library-support: publish alpha * ci-library-support: switch to alpha * ci-library-support: drop an info message in reduce
emmeowzing · Sep 26, 2024 · 8cf004b · 8cf004b
1 parent 3ec3cc7
commit 8cf004b
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 50 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,8 +6,8 @@ setup: true
 orbs:
   orb-tools: circleci/[email protected]
   circleci-cli: circleci/[email protected]
-  # dynamic: bjd2385/dynamic-continuation@dev:alpha
-  dynamic: bjd2385/[email protected]
+  dynamic: bjd2385/dynamic-continuation@dev:alpha
+  # dynamic: bjd2385/[email protected]
   general: premiscale/[email protected]
   slack: circleci/[email protected]
 
@@ -18,6 +18,7 @@ workflows:
       # Development branches
 
       - dynamic/continue:
+          library-config: library-config
           context: orb-publishing
 
       # On tag

diff --git a/.circleci/library-config.yml b/.circleci/library-config.yml
@@ -0,0 +1,46 @@
+jobs:
+  bats-tests:
+    parameters:
+      yq-version:
+        description: |+
+          Version of yq to install.
+
+          https://github.com/mikefarah/yq/releases
+        type: string
+        default: 4.40.5
+      tests-directory:
+        description: Directory in which bats tests reside.
+        type: string
+        default: tests/
+      formatter:
+        description: Bats formatter.
+        type: string
+        default: tap
+      resource-class:
+        description: Resource class to execute as.
+        type: enum
+        default: small
+        enum:
+          - small
+          - medium
+          - large
+          - 2xlarge
+    executor: default
+    resource_class: << parameters.resource-class >>
+    steps:
+      - checkout
+      - run:
+          name: Install bats
+          command: |+
+            sudo apt update
+            sudo apt install -y bats
+      - run:
+          name: Install yq
+          command: |+
+            wget https://github.com/mikefarah/yq/releases/download/v<< parameters.yq-version >>/yq_linux_amd64 -o yq
+            sudo install yq /usr/bin/yq
+            rm yq
+      - run:
+          name: Bats tests
+          command: |+
+            bats --formatter << parameters.formatter >> --timing --recursive << parameters.tests-directory >>
diff --git a/.circleci/src.yml b/.circleci/src.yml
@@ -14,52 +14,7 @@ executors:
       - image: cimg/base:2024.09
 
 
-jobs:
-  bats-tests:
-    parameters:
-      yq-version:
-        description: |+
-          Version of yq to install.
-
-          https://github.com/mikefarah/yq/releases
-        type: string
-        default: 4.40.5
-      tests-directory:
-        description: Directory in which bats tests reside.
-        type: string
-        default: tests/
-      formatter:
-        description: Bats formatter.
-        type: string
-        default: tap
-      resource-class:
-        description: Resource class to execute as.
-        type: enum
-        default: small
-        enum:
-          - small
-          - medium
-          - large
-          - 2xlarge
-    executor: default
-    resource_class: << parameters.resource-class >>
-    steps:
-      - checkout
-      - run:
-          name: Install bats
-          command: |+
-            sudo apt update
-            sudo apt install -y bats
-      - run:
-          name: Install yq
-          command: |+
-            wget https://github.com/mikefarah/yq/releases/download/v<< parameters.yq-version >>/yq_linux_amd64 -o yq
-            sudo install yq /usr/bin/yq
-            rm yq
-      - run:
-          name: Bats tests
-          command: |+
-            bats --formatter << parameters.formatter >> --timing --recursive << parameters.tests-directory >>
+jobs: {}
 
 
 workflows:

diff --git a/scripts/circleci-config-validate.sh b/scripts/circleci-config-validate.sh
@@ -8,12 +8,20 @@ if [[ -n $CIRCLECI ]]; then
 fi
 
 if ! command -v circleci &>/dev/null; then
-    echo "Circleci CLI could not be found. Install the latest CLI version https://circleci.com/docs/2.0/local-cli/#installation"
+    echo "Circleci CLI could not be found. Install the latest CLI version: https://circleci.com/docs/2.0/local-cli/#installation"
     exit 1
 fi
 
+if ! command -v yq &>/dev/null; then
+    echo "yq could not be found. Install the latest yq version: https://github.com/mikefarah/yq/releases"
+    exit 1
+fi
+
+
+lib="$(grep -oP "(?<=library-config: ).*" .circleci/config.yml)"
+
 for config in "$@"; do
-    if ! reMSG=$(circleci config validate --skip-update-check -c "$config"); then
+    if ! reMSG=$( circleci config validate --skip-update-check -c <(yq -Mr eval-all "explode(.) as \$item ireduce ( {}; . * \$item )" <(printf "%s\\n%s" "$config" "$lib" ) ) ); then
         printf "CircleCI config file \"%s\" failed validation.\\n" "$config"
         echo "${reMSG}"
         exit 1

diff --git a/src/commands/filter.yml b/src/commands/filter.yml
@@ -37,6 +37,13 @@ parameters:
     description: Provides the ability to map root repository changes (./) to a config file name.
     type: string
     default: app
+  library-config:
+    description: |+
+      Specify a common configuration file that serves as a library of commands and jobs.
+
+      This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution.
+    type: string
+    default: cilib
   reporting-window:
     description: The time window used to calculate summary metrics for the default branch of the repository. Defaults to disabled. Allows users to force all workflows to run if no workflows have been ran in the time window.
     type: enum
@@ -83,6 +90,7 @@ steps:
         SH_DYNAMIC_CONTINUATION_DEBUG: << parameters.debug >>
         SH_FORCE_ALL: << parameters.force-all >>
         SH_INCLUDE_CONFIG_CHANGES: << parameters.include-config-changes >>
+        SH_LIBRARY_CONFIG: << parameters.library-config >>
         SH_MODULES: << parameters.modules >>
         SH_MODULES_FILTERED: << parameters.modules-filtered >>
         SH_PROJECT_TYPE: << parameters.project-type >>

diff --git a/src/commands/reduce.yml b/src/commands/reduce.yml
@@ -14,6 +14,13 @@ parameters:
     description: Provides the ability to map root repository changes (./) to a config file name. Defaults to app.yml.
     type: string
     default: app
+  library-config:
+    description: |+
+      Specify a common configuration file that serves as a library of commands and jobs.
+
+      This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution.
+    type: string
+    default: ''
   continue-config:
     description: Path to the internally-used config for continuation
     type: string
@@ -49,6 +56,7 @@ steps:
       environment:
         SH_CONTINUE_CONFIG: << parameters.continue-config >>
         SH_DYNAMIC_CONTINUATION_DEBUG: << parameters.debug >>
+        SH_LIBRARY_CONFIG: << parameters.library-config >>
         SH_MODULES_FILTERED: << parameters.modules-filtered >>
         SH_ROOT_CONFIG: << parameters.root-config >>
       command: << include(scripts/reduce.sh) >>
diff --git a/src/jobs/continue.yml b/src/jobs/continue.yml
@@ -74,6 +74,13 @@ parameters:
     default: .circleci/continue-config.yml
 
   # both
+  library-config:
+    description: |+
+      Specify a common configuration file that serves as a library of commands and jobs.
+
+      This config is not validated, but rather merged with all other configs (excluding .circleci/config.yml, of course) prior to their validation and execution.
+    type: string
+    default: ''
   root-config:
     description: Provides the ability to map root repository changes (./) to a config file name. Name should be left without extension.
     type: string
@@ -164,6 +171,7 @@ steps:
       force-all: << parameters.force-all >>
       base-revision: << parameters.base-revision >>
       root-config: << parameters.root-config >>
+      library-config: << parameters.library-config >>
       reporting-window: << parameters.reporting-window >>
       circle-organization: << parameters.circle-organization >>
       circle-token: << parameters.circle-token >>
@@ -178,6 +186,7 @@ steps:
       modules-filtered: << parameters.modules-filtered >>
       continue-config: << parameters.continue-config >>
       root-config: << parameters.root-config >>
+      library-config: << parameters.library-config >>
       circle-token: << parameters.circle-token >>
       project-type: << parameters.project-type >>
       cache: false

diff --git a/src/scripts/filter.sh b/src/scripts/filter.sh
@@ -71,6 +71,7 @@ while [ "${SH_CIRCLE_ORGANIZATION:0:1}" = '$' ]; do
     SH_CIRCLE_ORGANIZATION="$(eval echo "$SH_CIRCLE_ORGANIZATION")"
 done
 
+
 # CircleCI API token should be set.
 if [ -z "$SH_CIRCLE_TOKEN" ]; then
     error "must set CircleCI token for successful authentication."
@@ -81,10 +82,12 @@ fi
 # Move yaml files -> yml so we can handle both extensions for YAML configs. Not that we want both, but we should handle both cases.
 for f in .circleci/*.yaml; do
     warn "migrating pipeline \"$f\" -> \"${f%.*}.yml\""
+
     if [ -f "${f%.*}.yml" ]; then
         error "could not migrate \"$f\", \"${f%.*}.yml\" already exists."
         exit 1
     fi
+
     mv "$f" "${f%.*}.yml"
 done
 

diff --git a/src/scripts/reduce.sh b/src/scripts/reduce.sh
@@ -1,6 +1,20 @@
 # shellcheck disable=SC2288,SC2001,SC2148,SC2002,SC2016,SC2046
 
 
+info()
+{
+    if [ $# -ne 1 ]; then
+        printf "Function \"info\" expected at least 1 argument: info message.\\n" >&2
+        exit 1
+    fi
+
+    local msg
+    msg="$1"
+
+    printf "INFO: %s\\n" "$msg"
+}
+
+
 # If `modules` is unavailable, stop this job without continuation
 if [ ! -f "$SH_MODULES_FILTERED" ] || [ ! -s "$SH_MODULES_FILTERED" ]; then
     printf "Nothing to merge. Halting the job.\\n"
@@ -19,4 +33,10 @@ awk "{
 }" "$SH_MODULES_FILTERED" > /tmp/"$CIRCLE_WORKFLOW_ID.txt"
 mv /tmp/"$CIRCLE_WORKFLOW_ID.txt" "$SH_MODULES_FILTERED"
 
+# Append the library config, if it is specified and exists, to the reduction.
+if [ "$SH_LIBRARY_CONFIG" != "" ] && [ -f .circleci/"$SH_LIBRARY_CONFIG".yml ]; then
+    info "Merging user-specified library config \"%s\" (.circleci/%s.yml) with final config.\\n" "$SH_LIBRARY_CONFIG"
+    printf ".circleci/%s.yml" "$SH_LIBRARY_CONFIG" >> "$SH_MODULES_FILTERED"
+fi
+
 yq -Mr eval-all 'explode(.) as $item ireduce ( {}; . * $item )' $(cat "$SH_MODULES_FILTERED" | xargs) | tee "$SH_CONTINUE_CONFIG"