support for CircleCI

README.md

@@ -8,6 +8,7 @@ Currently supported formats are
 [GitLab CI](https://docs.gitlab.com/ee/ci/yaml/gitlab_ci_yaml.html),
 [GitHub Actions](https://docs.github.com/en/actions),
 [Drone CI](https://docs.drone.io/pipeline/overview/),
+[CircleCI](https://circleci.com/docs/2.0/configuration-reference/),
 and (very limited) [Ansible](https://docs.ansible.com/ansible/2.9/modules/shell_module.html)
 
 ## Usage
@@ -96,6 +97,13 @@ Drone CI has a very simple file structure. As far as I can tell it only has
 conditions, but no deep nesting or includes. All command lists are concatenated
 and checked as a shell script.
 
+### CircleCI
+
+CircleCI has many options, but fortunately the nesting of its `jobs.*.steps.run` script elements is straightforward. All command lists are concatenated and checked as a shell script.
+
+* `shell` attributes are supported  
+* [pipeline values and parameters](https://circleci.com/docs/2.0/pipeline-variables/) are ignored and replaced with placeholder shell variables
+
 ### GitLab CI Pipelines
 
 GitLab CI files have more structure, and we try to support more of it.

test-input/circleci-config.yaml

@@ -0,0 +1,133 @@
+# from https://circleci.com/docs/2.0/configuration-reference/#example-full-configuration
+version: 2.1
+jobs:
+  build:
+    docker:
+      - image: ubuntu:14.04
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+
+      - image: mongo:2.6.8
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+        command: [mongod, --smallfiles]
+
+      - image: postgres:9.4.1
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+        # some containers require setting environment variables
+        environment:
+          POSTGRES_USER: root
+
+      - image: redis@sha256:54057dd7e125ca41afe526a877e8bd35ec2cdd33b9217e022ed37bdcf7d09673
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+
+      - image: rabbitmq:3.5.4
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+
+    environment:
+      TEST_REPORTS: /tmp/test-reports
+
+    working_directory: ~/my-project
+
+    steps:
+      - checkout
+
+      - run:
+          shell: /bin/dash
+          command: echo 127.0.0.1 devhost | sudo tee -a /etc/hosts
+
+      # Create Postgres users and database
+      # Note the YAML heredoc '|' for nicer formatting
+      - run: |
+          sudo -u root createuser -h localhost --superuser ubuntu &&
+          sudo createdb -h localhost test_db
+
+      - restore_cache:
+          keys:
+            - v1-my-project-{{ checksum "project.clj" }}
+            - v1-my-project-
+
+      - run:
+          environment:
+            SSH_TARGET: "localhost"
+            TEST_ENV: "linux"
+          command: |
+            set -xu
+            mkdir -p ${TEST_REPORTS}
+            run-tests.sh
+            cp out/tests/*.xml ${TEST_REPORTS}
+
+      - run: |
+          set -xu
+          mkdir -p /tmp/artifacts
+          create_jars.sh << pipeline.number >>
+          cp *.jar /tmp/artifacts
+
+      - save_cache:
+          key: v1-my-project-{{ checksum "project.clj" }}
+          paths:
+            - ~/.m2
+
+      # Save artifacts
+      - store_artifacts:
+          path: /tmp/artifacts
+          destination: build
+
+      # Upload test results
+      - store_test_results:
+          path: /tmp/test-reports
+
+  deploy-stage:
+    docker:
+      - image: ubuntu:14.04
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+    working_directory: /tmp/my-project
+    steps:
+      - run:
+          name: Deploy if tests pass and branch is Staging
+          command: ansible-playbook site.yml -i staging
+
+  deploy-prod:
+    docker:
+      - image: ubuntu:14.04
+        auth:
+          username: mydockerhub-user
+          password: $DOCKERHUB_PASSWORD  # context / project UI env-var reference
+    working_directory: /tmp/my-project
+    steps:
+      - run:
+          name: Deploy if tests pass and branch is Main
+          command: ansible-playbook site.yml -i production
+
+workflows:
+  version: 2
+  build-deploy:
+    jobs:
+      - build:
+          filters:
+            branches:
+              ignore:
+                - develop
+                - /feature-.*/
+      - deploy-stage:
+          requires:
+            - build
+          filters:
+            branches:
+              only: staging
+      - deploy-prod:
+          requires:
+            - build
+          filters:
+            branches:
+              only: main

yaml_shellcheck.py

@@ -157,6 +157,58 @@ def get_runs(data, path):
     return result
 
 
+def get_circleci_scripts(data):
+    """CircleCI: match on `jobs.*.steps.run`,
+    https://circleci.com/docs/2.0/configuration-reference/
+    """
+
+    result = {}
+    if "jobs" not in data:
+        return result
+    for jobkey, job in data["jobs"].items():
+        steps = job.get("steps", [])
+        logging.debug("job %s: %s", jobkey, steps)
+        for step_num, step in enumerate(steps):
+            if not (isinstance(step, dict) and "run" in step):
+                logging.debug("job %s, step %d: no run declaration", jobkey, step_num)
+                continue
+            run = step["run"]
+            shell = None
+            logging.debug("job %s, step %d: found %s %s", jobkey, step_num, type(run), run)
+            # challenge: the run element can have different data types
+            if isinstance(run, dict):
+                if "command" in run:
+                    script = run["command"]
+                    if "shell" in run:
+                        shell = run["shell"]
+                else:
+                    pass  # could be a directive like `save_cache`
+            elif isinstance(run, str):
+                script = run
+            elif isinstance(run, list):
+                script = "\n".join(run)
+            else:
+                raise ValueError(f"unexpected data type {type(run)} in job {jobkey} step {step_num}")
+
+            # CircleCI uses '<< foo >>' for context parameters,
+            # we try to be useful and replace these with a simple shell variable
+            script = re.sub(r'<<\s*([^\s>]*)\s*>>', r'"$PARAMETER"', script)
+            # add shebang line if we saw a 'shell' attribute
+            # TODO: we do not check for supported shell like we do in get_ansible_scripts
+            # TODO: not sure what is the best handling of bash vs. sh as default here
+            if not shell:
+                # CircleCI default shell, see doc "Default shell options"
+                shell = "/bin/bash"
+
+            script = f"#!{shell}\n" + script
+            result[f"{jobkey}/{step_num}"] = script
+
+    logging.debug("got scripts: %s", result)
+    for key in result:
+        logging.debug("%s: %s", key, result[key])
+    return result
+
+
 def get_drone_scripts(data):
     """Drone CI has a simple file format, with all scripts in
     `lists in steps[].commands[]`, see https://docs.drone.io/yaml/exec/
@@ -262,6 +314,9 @@ def select_yaml_schema(data, filename):
     elif isinstance(data, dict) and "on" in data and "jobs" in data:
         logging.info(f"read {filename} as GitHub Actions config...")
         return get_github_scripts
+    elif isinstance(data, dict) and "version" in data and "jobs" in data:
+        logging.info(f"read {filename} as CircleCI config...")
+        return get_circleci_scripts
     elif (
         isinstance(data, dict) and "steps" in data and "kind" in data and "type" in data
     ):
@@ -375,4 +430,4 @@ def cleanup_files(args):
     check_proc_result = run_shellcheck(args, filenames)
     cleanup_files(args)
     # exit with shellcheck exit code
-    sys.exit(check_proc_result.returncode)
+    sys.exit(check_proc_result.returncode if check_proc_result else 0)