From 1068fb4c6d0b4289c63d9eb241e371437e146408 Mon Sep 17 00:00:00 2001 From: DeveloperC Date: Wed, 4 Sep 2024 12:11:56 +0100 Subject: [PATCH] ci: adding Continuous integration (CI) workflow --- .github/workflows/continuous-integration.yml | 65 +++++++ .yamlfmt | 3 + Earthfile | 125 ++++++++++++- Earthfile.rej | 184 +++++++++++++++++++ ci/check-e2e-formatting.sh | 6 - ci/check-python-formatting.sh | 6 + ci/check-rust-formatting.sh | 6 + ci/check-rust-linting.sh | 8 + ci/check-shell-formatting.sh | 6 + ci/check-shell-linting.sh | 6 + ci/check-yaml-formatting.sh | 6 + ci/compile.sh | 8 + ci/end-to-end-test.sh | 7 + ci/fix-e2e-formatting.sh | 6 - ci/fix-python-formatting.sh | 6 + ci/fix-rust-formatting.sh | 6 + ci/fix-shell-formatting.sh | 6 + ci/fix-yaml-formatting.sh | 6 + ci/unit-test.sh | 8 + 19 files changed, 457 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/continuous-integration.yml create mode 100644 .yamlfmt create mode 100644 Earthfile.rej delete mode 100755 ci/check-e2e-formatting.sh create mode 100755 ci/check-python-formatting.sh create mode 100755 ci/check-rust-formatting.sh create mode 100755 ci/check-rust-linting.sh create mode 100755 ci/check-shell-formatting.sh create mode 100755 ci/check-shell-linting.sh create mode 100755 ci/check-yaml-formatting.sh create mode 100755 ci/compile.sh create mode 100755 ci/end-to-end-test.sh delete mode 100755 ci/fix-e2e-formatting.sh create mode 100755 ci/fix-python-formatting.sh create mode 100755 ci/fix-rust-formatting.sh create mode 100755 ci/fix-shell-formatting.sh create mode 100755 ci/fix-yaml-formatting.sh create mode 100755 ci/unit-test.sh diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml new file mode 100644 index 0000000..442f21b --- /dev/null +++ b/.github/workflows/continuous-integration.yml @@ -0,0 +1,65 @@ +name: Continuous Integration (CI) + +on: pull_request + +env: + # Forcing Earthly to use colours, to make reading output easier. + FORCE_COLOR: 1 + +jobs: + formatting: + name: Formatting + runs-on: ubuntu-latest + strategy: + matrix: + language: [rust, python, shell, yaml] + steps: + - name: Download Earthly v0.8.12. + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.12/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - name: Checkout code. + uses: actions/checkout@v3 + - name: Check formatting. + run: earthly --ci +check-${{ matrix.language }}-formatting + linting: + name: Linting + runs-on: ubuntu-latest + strategy: + matrix: + language: [rust, shell] + steps: + - name: Download Earthly v0.8.12. + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.12/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - name: Checkout code. + uses: actions/checkout@v3 + - name: Check linting. + run: earthly --ci +check-${{ matrix.language }}-linting + compile: + name: Compile + runs-on: ubuntu-latest + steps: + - name: Download Earthly v0.8.12. + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.12/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - name: Checkout code. + uses: actions/checkout@v3 + - name: Compile. + run: earthly --ci +compile + unit-test: + name: Unit Test + runs-on: ubuntu-latest + steps: + - name: Download Earthly v0.8.12. + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.12/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - name: Checkout code. + uses: actions/checkout@v3 + - name: Unit test. + run: earthly --ci +unit-test + end-to-end-test: + name: End to End Test + runs-on: ubuntu-latest + steps: + - name: Download Earthly v0.8.12. + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.12/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - name: Checkout code. + uses: actions/checkout@v3 + - name: End to End test. + run: earthly --ci +end-to-end-test diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 0000000..e9d24f9 --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,3 @@ +formatter: + type: basic + retain_line_breaks: true diff --git a/Earthfile b/Earthfile index cd1a687..41a1977 100644 --- a/Earthfile +++ b/Earthfile @@ -13,7 +13,9 @@ COPY_METADATA: rust-base: - FROM rust:1.70.0 + FROM rust:1.70.0-alpine3.18 + RUN apk add --no-cache musl-dev bash + WORKDIR "/clean_git_history" check-clean-git-history: @@ -32,12 +34,125 @@ check-conventional-commits-linting: RUN ./ci/check-conventional-commits-linting.sh --from-reference "${from_reference}" +COPY_SOURCECODE: + COMMAND + DO +COPY_CI_DATA + COPY --if-exists "Cargo.lock" "./" + COPY --dir "Cargo.toml" "clean_git_history/" "clean_git_history_lib/" "./" + + +rust-formatting-base: + FROM +rust-base + RUN rustup component add rustfmt + DO +COPY_SOURCECODE + + +check-rust-formatting: + FROM +rust-formatting-base + RUN ./ci/check-rust-formatting.sh + + +python-base: + FROM python:3.9.19-alpine + RUN apk add --no-cache git bash + WORKDIR "/consistent_whitespace" + DO +COPY_SOURCECODE + + +python-formatting-base: + FROM +python-base + RUN pip3 install -r "clean_git_history/end-to-end-tests/autopep8.requirements.txt" + + +check-python-formatting: + FROM +python-formatting-base + RUN ./ci/check-python-formatting.sh + + golang-base: FROM golang:1.22.1 - ENV GOPROXY=direct - ENV CGO_ENABLED=0 - ENV GOOS=linux - ENV GOARCH=amd64 + WORKDIR "/consistent_whitespace" + + +shell-formatting-base: + FROM +golang-base + RUN go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0 + DO +COPY_CI_DATA + + +check-shell-formatting: + FROM +shell-formatting-base + RUN ./ci/check-shell-formatting.sh + + +yaml-formatting-base: + FROM +golang-base + RUN go install github.com/google/yamlfmt/cmd/yamlfmt@v0.10.0 + COPY ".yamlfmt" "./" + DO +COPY_CI_DATA + + +check-yaml-formatting: + FROM +yaml-formatting-base + RUN ./ci/check-yaml-formatting.sh + + +check-formatting: + BUILD +check-rust-formatting + BUILD +check-python-formatting + BUILD +check-shell-formatting + BUILD +check-yaml-formatting + + +fix-rust-formatting: + FROM +rust-formatting-base + RUN ./ci/fix-rust-formatting.sh + SAVE ARTIFACT "src/" AS LOCAL "./" + + +fix-python-formatting: + FROM +python-formatting-base + RUN ./ci/fix-python-formatting.sh + SAVE ARTIFACT "end-to-end-tests/" AS LOCAL "./" + + +fix-shell-formatting: + FROM +shell-formatting-base + RUN ./ci/fix-shell-formatting.sh + SAVE ARTIFACT "ci/" AS LOCAL "./" + + +fix-yaml-formatting: + FROM +yaml-formatting-base + RUN ./ci/fix-yaml-formatting.sh + SAVE ARTIFACT ".github/" AS LOCAL "./" + + +fix-formatting: + BUILD +fix-rust-formatting + BUILD +fix-python-formatting + BUILD +fix-shell-formatting + BUILD +fix-yaml-formatting + + +check-rust-linting: + FROM +rust-base + RUN rustup component add clippy + DO +COPY_SOURCECODE + RUN ./ci/check-rust-linting.sh + + +ubuntu-base: + FROM ubuntu:22.04 + # https://askubuntu.com/questions/462690/what-does-apt-get-fix-missing-do-and-when-is-it-useful + RUN apt-get update --fix-missing + + +check-shell-linting: + FROM +ubuntu-base + RUN apt-get install shellcheck -y + DO +COPY_CI_DATA + RUN ./ci/check-shell-linting.sh check-github-actions-workflows-linting: diff --git a/Earthfile.rej b/Earthfile.rej new file mode 100644 index 0000000..e9eb82a --- /dev/null +++ b/Earthfile.rej @@ -0,0 +1,184 @@ +--- Earthfile ++++ Earthfile +@@ -13,7 +13,9 @@ COPY_METADATA: + + + rust-base: +- FROM rust:1.70.0 ++ FROM rust:1.70.0-alpine3.18 ++ RUN apk add --no-cache musl-dev bash ++ WORKDIR "/clean_git_history" + + + check-clean-git-history: +@@ -32,12 +34,125 @@ check-conventional-commits-linting: + RUN ./ci/check-conventional-commits-linting.sh --from-reference "${from_reference}" + + ++COPY_SOURCECODE: ++ COMMAND ++ DO +COPY_CI_DATA ++ COPY --if-exists "Cargo.lock" "./" ++ COPY --dir "Cargo.toml" "clean_git_history/" "clean_git_history_lib/" "./" ++ ++ ++rust-formatting-base: ++ FROM +rust-base ++ RUN rustup component add rustfmt ++ DO +COPY_SOURCECODE ++ ++ ++check-rust-formatting: ++ FROM +rust-formatting-base ++ RUN ./ci/check-rust-formatting.sh ++ ++ ++python-base: ++ FROM python:3.9.19-alpine ++ RUN apk add --no-cache git bash ++ WORKDIR "/consistent_whitespace" ++ DO +COPY_SOURCECODE ++ ++ ++python-formatting-base: ++ FROM +python-base ++ RUN pip3 install -r "clean_git_history/end-to-end-tests/autopep8.requirements.txt" ++ ++ ++check-python-formatting: ++ FROM +python-formatting-base ++ RUN ./ci/check-python-formatting.sh ++ ++ + golang-base: + FROM golang:1.22.1 +- ENV GOPROXY=direct +- ENV CGO_ENABLED=0 +- ENV GOOS=linux +- ENV GOARCH=amd64 ++ WORKDIR "/consistent_whitespace" ++ ++ ++shell-formatting-base: ++ FROM +golang-base ++ RUN go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0 ++ DO +COPY_CI_DATA ++ ++ ++check-shell-formatting: ++ FROM +shell-formatting-base ++ RUN ./ci/check-shell-formatting.sh ++ ++ ++yaml-formatting-base: ++ FROM +golang-base ++ RUN go install github.com/google/yamlfmt/cmd/yamlfmt@v0.10.0 ++ COPY ".yamlfmt" "./" ++ DO +COPY_CI_DATA ++ ++ ++check-yaml-formatting: ++ FROM +yaml-formatting-base ++ RUN ./ci/check-yaml-formatting.sh ++ ++ ++check-formatting: ++ BUILD +check-rust-formatting ++ BUILD +check-python-formatting ++ BUILD +check-shell-formatting ++ BUILD +check-yaml-formatting ++ ++ ++fix-rust-formatting: ++ FROM +rust-formatting-base ++ RUN ./ci/fix-rust-formatting.sh ++ SAVE ARTIFACT "src/" AS LOCAL "./" ++ ++ ++fix-python-formatting: ++ FROM +python-formatting-base ++ RUN ./ci/fix-python-formatting.sh ++ SAVE ARTIFACT "end-to-end-tests/" AS LOCAL "./" ++ ++ ++fix-shell-formatting: ++ FROM +shell-formatting-base ++ RUN ./ci/fix-shell-formatting.sh ++ SAVE ARTIFACT "ci/" AS LOCAL "./" ++ ++ ++fix-yaml-formatting: ++ FROM +yaml-formatting-base ++ RUN ./ci/fix-yaml-formatting.sh ++ SAVE ARTIFACT ".github/" AS LOCAL "./" ++ ++ ++fix-formatting: ++ BUILD +fix-rust-formatting ++ BUILD +fix-python-formatting ++ BUILD +fix-shell-formatting ++ BUILD +fix-yaml-formatting ++ ++ ++check-rust-linting: ++ FROM +rust-base ++ RUN rustup component add clippy ++ DO +COPY_SOURCECODE ++ RUN ./ci/check-rust-linting.sh ++ ++ ++ubuntu-base: ++ FROM ubuntu:22.04 ++ # https://askubuntu.com/questions/462690/what-does-apt-get-fix-missing-do-and-when-is-it-useful ++ RUN apt-get update --fix-missing ++ ++ ++check-shell-linting: ++ FROM +ubuntu-base ++ RUN apt-get install shellcheck -y ++ DO +COPY_CI_DATA ++ RUN ./ci/check-shell-linting.sh + + + check-github-actions-workflows-linting: +@@ -48,22 +163,28 @@ check-github-actions-workflows-linting: + + + check-linting: ++ BUILD +check-rust-linting ++ BUILD +check-shell-linting + BUILD +check-github-actions-workflows-linting + + +-e2e-formatting-base: +- FROM python:3.12.0-slim +- DO +COPY_CI_DATA +- COPY "clean_git_history/end-to-end-tests/" "./" +- RUN pip3 install -r "./clean_git_history/end-to-end-tests/autopep8.requirements.txt" ++compile: ++ FROM +rust-base ++ DO +COPY_SOURCECODE ++ RUN ./ci/compile.sh ++ SAVE ARTIFACT "target/" AS LOCAL "./" ++ SAVE ARTIFACT "Cargo.lock" AS LOCAL "./" + + +-check-e2e-formatting: +- FROM +e2e-formatting-base +- RUN ./ci/check-e2e-formatting.sh ++unit-test: ++ FROM +rust-base ++ DO +COPY_METADATA ++ DO +COPY_SOURCECODE ++ RUN ./ci/unit-test.sh + + +-fix-e2e-formatting: +- FROM +e2e-formatting-base +- RUN ./ci/fix-e2e-formatting.sh +- SAVE ARTIFACT "clean_git_history/end-to-end-tests/" AS LOCAL "./" ++end-to-end-test: ++ FROM +python-base ++ RUN pip3 install -r "clean_git_history/end-to-end-tests/requirements.txt" ++ COPY "+compile/target/" "target/" ++ RUN ./ci/end-to-end-test.sh diff --git a/ci/check-e2e-formatting.sh b/ci/check-e2e-formatting.sh deleted file mode 100755 index ff0c8ab..0000000 --- a/ci/check-e2e-formatting.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env sh - -set -o errexit -set -o xtrace - -find "./conventional_commits_linter/end-to-end-tests/features" -type f -name "*.py" | xargs -I {} autopep8 --exit-code --diff --aggressive --aggressive --max-line-length 120 "{}" diff --git a/ci/check-python-formatting.sh b/ci/check-python-formatting.sh new file mode 100755 index 0000000..806f746 --- /dev/null +++ b/ci/check-python-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +find "clean_git_history/end-to-end-tests/features/" -type f | grep "[.]py$" | xargs -I {} autopep8 --exit-code --diff --aggressive --aggressive --max-line-length 120 "{}" diff --git a/ci/check-rust-formatting.sh b/ci/check-rust-formatting.sh new file mode 100755 index 0000000..d2b81a7 --- /dev/null +++ b/ci/check-rust-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cargo fmt --all -- --check --config=group_imports=StdExternalCrate diff --git a/ci/check-rust-linting.sh b/ci/check-rust-linting.sh new file mode 100755 index 0000000..6270ef7 --- /dev/null +++ b/ci/check-rust-linting.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cargo clippy --verbose --workspace --all-targets -- -D warnings +cargo clippy --verbose --workspace --all-targets --all-features -- -D warnings +cargo clippy --verbose --workspace --all-targets --no-default-features -- -D warnings diff --git a/ci/check-shell-formatting.sh b/ci/check-shell-formatting.sh new file mode 100755 index 0000000..99dd161 --- /dev/null +++ b/ci/check-shell-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +shfmt --simplify --diff ./ci/* diff --git a/ci/check-shell-linting.sh b/ci/check-shell-linting.sh new file mode 100755 index 0000000..772cf87 --- /dev/null +++ b/ci/check-shell-linting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +shellcheck ci/* diff --git a/ci/check-yaml-formatting.sh b/ci/check-yaml-formatting.sh new file mode 100755 index 0000000..d39cb5a --- /dev/null +++ b/ci/check-yaml-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -o errexit +set -o xtrace + +yamlfmt -lint -dstar .github/**/* diff --git a/ci/compile.sh b/ci/compile.sh new file mode 100755 index 0000000..c08c588 --- /dev/null +++ b/ci/compile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cargo build --verbose --workspace +cargo build --verbose --workspace --all-features +cargo build --verbose --workspace --no-default-features diff --git a/ci/end-to-end-test.sh b/ci/end-to-end-test.sh new file mode 100755 index 0000000..b2b25d0 --- /dev/null +++ b/ci/end-to-end-test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cd "clean_git_history/end-to-end-tests/" +behave diff --git a/ci/fix-e2e-formatting.sh b/ci/fix-e2e-formatting.sh deleted file mode 100755 index ee1c5b4..0000000 --- a/ci/fix-e2e-formatting.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env sh - -set -o errexit -set -o xtrace - -find "./conventional_commits_linter/end-to-end-tests/features" -type f -name "*.py" | xargs -I {} autopep8 --in-place --aggressive --aggressive --max-line-length 120 "{}" diff --git a/ci/fix-python-formatting.sh b/ci/fix-python-formatting.sh new file mode 100755 index 0000000..e4af39e --- /dev/null +++ b/ci/fix-python-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +find "clean_git_history/end-to-end-tests/features/" -type f | grep "[.]py$" | xargs -I {} autopep8 --in-place --aggressive --aggressive --max-line-length 120 "{}" diff --git a/ci/fix-rust-formatting.sh b/ci/fix-rust-formatting.sh new file mode 100755 index 0000000..5053985 --- /dev/null +++ b/ci/fix-rust-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cargo fmt --all -- --config=group_imports=StdExternalCrate diff --git a/ci/fix-shell-formatting.sh b/ci/fix-shell-formatting.sh new file mode 100755 index 0000000..a183ba5 --- /dev/null +++ b/ci/fix-shell-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +shfmt --simplify --write ./ci/* diff --git a/ci/fix-yaml-formatting.sh b/ci/fix-yaml-formatting.sh new file mode 100755 index 0000000..7f5c712 --- /dev/null +++ b/ci/fix-yaml-formatting.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +yamlfmt -dstar ./github/**/* diff --git a/ci/unit-test.sh b/ci/unit-test.sh new file mode 100755 index 0000000..043f87d --- /dev/null +++ b/ci/unit-test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +set -o errexit +set -o xtrace + +cargo test --verbose --workspace +cargo test --verbose --workspace --all-features +cargo test --verbose --workspace --no-default-features