From 350684ed9b7b9de67a6c78f98492b2f8bd836856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Sun, 31 Oct 2021 11:30:27 +0100 Subject: [PATCH 01/11] feat(salt-api): Add SALT_API_CERT_CN env variable Use this environment variable in order to set the salt-api certificatescommon name. --- Dockerfile | 2 +- README.md | 1 + assets/runtime/env-defaults.sh | 1 + assets/runtime/functions.sh | 11 +++++------ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 335e6781..6d3582e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ARG VCS_REF ENV SALT_VERSION="3004" \ PYTHON_VERSION="3.8" -ENV IMAGE_VERSION="${SALT_VERSION}" +ENV IMAGE_VERSION="${SALT_VERSION}_1" ENV SALT_DOCKER_DIR="/etc/docker-salt" \ SALT_ROOT_DIR="/etc/salt" \ diff --git a/README.md b/README.md index a516bed2..fff4053b 100644 --- a/README.md +++ b/README.md @@ -507,6 +507,7 @@ Below you can find a list with the available options that can be used to customi | `SALT_API_SERVICE_ENABLED` | Enable `salt-api` service. Default: `false` | | `SALT_API_USER` | Set username for `salt-api` service. Default: `salt_api` | | `SALT_API_USER_PASS` | `SALT_API_USER` password. Required if `SALT_API_SERVICE_ENBALED` is `true` and `SALT_API_USER` is not empty. _Unset_ by default | +| `SALT_API_CERT_CN` | Common name in the request. Default: `localhost` | | `SALT_MASTER_SIGN_PUBKEY` | Sign the master auth-replies with a cryptographic signature of the master's public key. Possible values: 'True' or 'False'. Default: `False` | | `SALT_MASTER_USE_PUBKEY_SIGNATURE` | Instead of computing the signature for each auth-reply, use a pre-calculated signature. This option requires `SALT_MASTER_SIGN_PUBKEY` set to 'True'. Possible values: 'True' or 'False'. Default: `True` | | `SALT_MASTER_SIGN_KEY_NAME` | The customizable name of the signing-key-pair without suffix. Default: `master_sign` | diff --git a/assets/runtime/env-defaults.sh b/assets/runtime/env-defaults.sh index bae2f93d..40ccdbf0 100755 --- a/assets/runtime/env-defaults.sh +++ b/assets/runtime/env-defaults.sh @@ -5,6 +5,7 @@ TIMEZONE=${TIMEZONE:-UTC} SALT_API_SERVICE_ENABLED=${SALT_API_SERVICE_ENABLED:-false} SALT_API_USER=${SALT_API_USER:-salt_api} +SALT_API_CERT_CN=${SALT_API_CERT_CN:-localhost} SALT_LOG_ROTATE_FREQUENCY=${SALT_LOG_ROTATE_FREQUENCY:-weekly} SALT_LOG_ROTATE_RETENTION=${SALT_LOG_ROTATE_RETENTION:-52} diff --git a/assets/runtime/functions.sh b/assets/runtime/functions.sh index 15118802..af7678f0 100755 --- a/assets/runtime/functions.sh +++ b/assets/runtime/functions.sh @@ -208,10 +208,9 @@ function configure_salt_api() echo "Configuring salt-api service ..." CERTS_PATH=/etc/pki - SALT_API_KEY_FILE='docker-salt-master' - rm -rf "${CERTS_PATH}/tls/certs/*" - salt-call --local tls.create_self_signed_cert cacert_path="${CERTS_PATH}" CN="${SALT_API_KEY_FILE}" - chown "${SALT_USER}": "${CERTS_PATH}/tls/certs/${SALT_API_KEY_FILE}".{crt,key} + rm -rf "${CERTS_PATH}"/tls/certs/* + salt-call --local tls.create_self_signed_cert cacert_path="${CERTS_PATH}" CN="${SALT_API_CERT_CN}" + chown "${SALT_USER}": "${CERTS_PATH}/tls/certs/${SALT_API_CERT_CN}".{crt,key} cat >> "${SALT_ROOT_DIR}/master" < Date: Sun, 31 Oct 2021 11:31:33 +0100 Subject: [PATCH 02/11] fix(salt-api): Change supervisor priority Set lower priority for the salt-api service to ensure it starts after the salt-master process. --- assets/runtime/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/runtime/functions.sh b/assets/runtime/functions.sh index af7678f0..69e9363f 100755 --- a/assets/runtime/functions.sh +++ b/assets/runtime/functions.sh @@ -229,7 +229,7 @@ EOF # configure supervisord to start salt-api cat > /etc/supervisor/conf.d/salt-api.conf < Date: Sun, 31 Oct 2021 11:52:49 +0100 Subject: [PATCH 03/11] fix(shellcheck): Suppress SC1091 error --- assets/runtime/functions.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/assets/runtime/functions.sh b/assets/runtime/functions.sh index 69e9363f..69b75acf 100755 --- a/assets/runtime/functions.sh +++ b/assets/runtime/functions.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash set -e -source "${SALT_RUNTIME_DIR}/env-defaults.sh" + +# shellcheck source=assets/runtime/env-defaults.sh +ENV_DEFAULTS_FILE="${SALT_RUNTIME_DIR}/env-defaults.sh" +source "${ENV_DEFAULTS_FILE}" # cdalvaro managed block string SELF_MANAGED_BLOCK_STRING="## cdalvaro managed block" From d795ea2963a05acbf2f82f78621311e7a561274f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Sun, 31 Oct 2021 22:17:22 +0100 Subject: [PATCH 04/11] feat: Install pygit2 1.7.0 --- .github/workflows/build-and-test.yml | 9 ++ .gitignore | 8 +- CHANGELOG.md | 6 + Dockerfile | 2 +- README.md | 10 +- assets/build/functions.sh | 224 ++++++++++++++++++++++++++- assets/build/install.sh | 85 +++++----- tests/gitfs/README.md | 4 + tests/gitfs/config/fileserver.conf | 2 + tests/gitfs/config/gitfs.conf | 19 +++ tests/gitfs/test.sh | 41 +++++ tests/salt-api/test.sh | 26 ++-- 12 files changed, 383 insertions(+), 53 deletions(-) create mode 100644 tests/gitfs/README.md create mode 100644 tests/gitfs/config/fileserver.conf create mode 100644 tests/gitfs/config/gitfs.conf create mode 100755 tests/gitfs/test.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ed0685d3..7d74b685 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -119,6 +119,15 @@ jobs: - name: Execute salt-api tests run: tests/salt-api/test.sh + - name: Execute gitfs tests + env: + GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs + run: | + mkdir -p "${GITFS_KEYS_DIR}" + echo ${{ secrets.TESTS_REPO_PRIVATE_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$ + echo ${{ secrets.TESTS_REPO_PUBLIC_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$ + tests/gitfs/test.sh + - name: Cleanup run: | docker stop registry diff --git a/.gitignore b/.gitignore index d70f4093..8c922aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +# Always included +!assets/** + # development /config/ /roots/ /keys/ /logs/ /3pfs/ -.vscode/ \ No newline at end of file +.vscode/ + +# tests +/tests/**/keys/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b01faa7c..72be1a0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ This file only reflects the changes that are made in this image. Please refer to the [Salt 3004 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3004.html) for the list of changes in SaltStack. +**3004_1** + +- Install `libssh2 1.10.0` from source +- Install `libgit2 1.3.0` from source +- Install `pygit2 1.7.0` from pip repositories + **3004** - Upgrade `salt-master` to `3004` *Silicon* diff --git a/Dockerfile b/Dockerfile index 6d3582e2..b3bedcb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ WORKDIR ${SALT_BUILD_DIR} # hadolint ignore=DL3008 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - sudo ca-certificates apt-transport-https wget locales openssh-client \ + sudo ca-certificates openssl apt-transport-https wget locales openssh-client \ python${PYTHON_VERSION} python3-dev libpython3-dev \ python3-pip python3-setuptools python3-wheel \ supervisor logrotate git gettext-base tzdata \ diff --git a/README.md b/README.md index fff4053b..5bb017ad 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ This keys must be placed inside `/home/salt/data/keys` directory. You can create an ssh key for pygit2 with the following command: ```sh -ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com' +ssh-keygen -f gitfs_ssh -C 'gitfs@example.com' ``` Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file. @@ -334,13 +334,19 @@ gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub **Important Note** +By default, this image has been tested with RSA 4096 ssh keys generated with `ssh-keygen`. + If you get the following error while using `gitfs` with `pygit2` ```plain _pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request ``` -look if your private key hash empty lines at the bottom of the file and suppress them for solving the error. +you may have to recreate your ssh key adding the parameter: `-m PEM`: + +```sh +ssh-keygen -m PEM -f gitfs_ssh -C 'gitfs@example.com' +``` ### 3rd Party Formulas diff --git a/assets/build/functions.sh b/assets/build/functions.sh index 300ddea3..8254e0f9 100755 --- a/assets/build/functions.sh +++ b/assets/build/functions.sh @@ -2,7 +2,68 @@ set -e -# Execute a command as SALT_USER +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __detect_color_support +# DESCRIPTION: Try to detect color support. +#---------------------------------------------------------------------------------------------------------------------- +_COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} +__detect_color_support() { + # shellcheck disable=SC2181 + if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then + RC='\033[1;31m' + GC='\033[1;32m' + BC='\033[1;34m' + YC='\033[1;33m' + EC='\033[0m' + else + RC="" + GC="" + BC="" + YC="" + EC="" + fi +} +__detect_color_support + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_debug +# DESCRIPTION: Echo debug information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_debug() { + if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then + echo -e "${BC} * DEBUG${EC}: $*" + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_info +# DESCRIPTION: Echo information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_info() { + echo -e "${GC} * INFO${EC}: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_warn +# DESCRIPTION: Echo warning information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_warn() { + echo -e "${YC} * WARN${EC}: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_error +# DESCRIPTION: Echo errors to stderr. +#---------------------------------------------------------------------------------------------------------------------- +function log_error() +{ + (>&2 echo -e "${RC} * ERROR${EC}: $*") +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: exec_as_salt +# DESCRIPTION: Execute the pass command as the `salt` user. +#---------------------------------------------------------------------------------------------------------------------- function exec_as_salt() { if [[ $(whoami) == "${SALT_USER}" ]]; then @@ -11,3 +72,164 @@ function exec_as_salt() sudo -HEu "${SALT_USER}" "$@" fi } + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 32-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm32() +{ + uname -m | grep -qE 'armv7l' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 64-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm64() +{ + uname -m | grep -qE 'arm64|aarch64' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm() +{ + is_arm32 || is_arm64 +} + +function install_pkgs() +{ + apt-get install --no-install-recommends -y $@ +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: download +# DESCRIPTION: Download the content from the given URL and save it into the specified file. +#---------------------------------------------------------------------------------------------------------------------- +function download() +{ + local URL="$1" + local FILE_NAME="$2" + + local WGET_ARGS=(--quiet) + is_arm32 && WGET_ARGS+=(--no-check-certificate) + + log_info "Downloading ${FILE_NAME} from ${URL} ..." + wget ${WGET_ARGS[@]} -O "${FILE_NAME}" "${URL}" + if [[ -f "${FILE_NAME}" ]]; then + log_debug "Success!" + else + log_error "Failed to download ${URL}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: check_sha256 +# DESCRIPTION: Compute the SHA256 hash for the given file and check if it matches the expected one. +#---------------------------------------------------------------------------------------------------------------------- +function check_sha256() +{ + local FILE="${1}" + local SHA256="${2}" + + log_info "Checking ${FILE} SHA256 hash ..." + if echo "${SHA256} ${FILE}" | shasum -a 256 -c --status -; then + log_debug "SHA256 hash for ${FILE} matches! (${SHA256})" + else + local HASH=$(shasum -a 256 "${FILE}" | awk '{print $1}') + log_error "SHA256 checksum mismatch for ${FILE}" + log_error "Expected: ${SHA256}" + log_error " Got: ${HASH}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: extract +# DESCRIPTION: Extract the given .tar.gz into the current directory. +#---------------------------------------------------------------------------------------------------------------------- +function extract() +{ + local FILE="${1}" + log_info "Unpacking file: ${FILE}" + tar xzf "${FILE}" --strip-components 1 +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: build_and_install +# DESCRIPTION: Build and install the given package from the current directory using cmake. +#---------------------------------------------------------------------------------------------------------------------- +function build_and_install() +{ + local PACKAGE_NAME="${1}"; shift + local CMAKE_ARGS=( + -Wno-dev + -DCMAKE_BUILD_TYPE=Release + ) + + # shellcheck disable=SC2206 + CMAKE_ARGS+=( $@ ) + + log_info "Building and installing ${PACKAGE_NAME} ..." + log_debug "CMAKE_ARGS: ${CMAKE_ARGS[@]}" + cmake ${CMAKE_ARGS[@]} . + cmake --build . --target install --config Release +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_libssh2 +# DESCRIPTION: Install libssh2 library. +#---------------------------------------------------------------------------------------------------------------------- +function install_libssh2() +{ + local LIBSSH2_VERSION=1.10.0 + local LIBSSH2_URL="https://github.com/libssh2/libssh2/releases/download/libssh2-${LIBSSH2_VERSION}/libssh2-${LIBSSH2_VERSION}.tar.gz" + local FILE_NAME="libssh2-${LIBSSH2_VERSION}.tar.gz" + local SHA256_SUM='2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBSSH2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + _OPTS=( + -DLINT=OFF + -DBUILD_SHARED_LIBS=ON + -DCRYPTO_BACKEND=OpenSSL + -DENABLE_ZLIB_COMPRESSION=ON + -DENABLE_DEBUG_LOGGING=OFF + -DCLEAR_MEMORY=ON + ) + + build_and_install "libssh2 v${LIBSSH2_VERSION}" ${_OPTS[@]} + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} + +# Install libgit2 library +function install_libgit2() +{ + local LIBGIT2_VERSION=1.3.0 + local LIBGIT2_URL="https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz" + local FILE_NAME="libgit2-${LIBGIT2_VERSION}.tar.gz" + local SHA256_SUM='192eeff84596ff09efb6b01835a066f2df7cd7985e0991c79595688e6b36444e' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBGIT2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + build_and_install "libgit2 v${LIBGIT2_VERSION}" -DBUILD_CLAR=OFF + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} diff --git a/assets/build/install.sh b/assets/build/install.sh index 3ca59940..c47e71a5 100755 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -2,24 +2,28 @@ set -e -# shellcheck disable=SC1091 -source "${SALT_BUILD_DIR}/functions.sh" +export DEBIAN_FRONTEND=noninteractive -echo "Installing build dependencies ..." -BUILD_DEPENDENCIES=(make gcc g++ cmake pkg-config) +# shellcheck source=assets/build/functions.sh +FUNCTIONS_FILE="${SALT_BUILD_DIR}/functions.sh" +source "${FUNCTIONS_FILE}" -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y "${BUILD_DEPENDENCIES[@]}" +log_info "Installing required packages and build dependencies ..." +REQUIRED_PACKAGES=( + libssl1.1 zlib1g libffi7 libpcre3 libgssapi3-heimdal +) + +BUILD_DEPENDENCIES=( + make gcc g++ cmake pkg-config libssl-dev zlib1g-dev libffi-dev + libpcre3-dev heimdal-dev +) -# Install arm build dependencies -if [[ "$(uname -i)" =~ ^(arm|aarch64) ]]; then - echo "Installing arm specific dependencies ..." - DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - libzmq3-dev libhttp-parser-dev libssl-dev libcurl4-openssl-dev -fi +apt-get update +apt-get install --no-install-recommends -y \ + "${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}" # Create salt user -echo "Creating ${SALT_USER} user ..." +log_info "Creating ${SALT_USER} user ..." useradd -d "${SALT_HOME}" -ms /bin/bash -U -G root,sudo,shadow "${SALT_USER}" # Set PATH @@ -28,14 +32,28 @@ PATH=/usr/local/sbin:/usr/local/bin:\$PATH EOF # Install python3 packages -echo "Installing python3 packages ..." -DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - python3-mako python3-pycryptodome python3-cherrypy3 python3-git python3-requests \ - python3-redis python3-gnupg python3-mysqldb python3-dateutil python3-libnacl python3-openssl \ - python3-pygit2 +log_info "Installing python3 packages ..." +apt-get install --yes --quiet --no-install-recommends \ + python3-mako python3-pycryptodome python3-cherrypy3 \ + python3-git python3-requests python3-redis python3-gnupg \ + python3-mysqldb python3-dateutil python3-libnacl python3-openssl pip3 install timelib==0.2.5 +# Install pygit2 package +install_libssh2 +install_libgit2 +pip3 install pygit2==1.7.0 + +# Downloading bootstrap-salt.sh script +BOOTSTRAP_VERSION='2021.09.17' +BOOTSTRAP_URL="https://raw.githubusercontent.com/saltstack/salt-bootstrap/v${BOOTSTRAP_VERSION}/bootstrap-salt.sh" +BOOTSTRAP_FILE='bootstrap-salt.sh' +BOOTSTRAP_SHA256='090d652cd6290debce0e3a4eded65086a4272e69446e711eb26f87160593b6a2' + +download "${BOOTSTRAP_URL}" "${BOOTSTRAP_FILE}" +check_sha256 "${BOOTSTRAP_FILE}" "${BOOTSTRAP_SHA256}" + # Bootstrap script options: # https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options ## -M: install Salt Master by default @@ -46,22 +64,17 @@ pip3 install timelib==0.2.5 ## -p: Extra-package to install ## -x: Changes the python version used to install a git version of salt SALT_BOOTSTRAP_OPTS=( -M -N -X -d -P -p salt-api -p salt-call -x "python${PYTHON_VERSION}" ) -_WGET_ARGS=() - -if [[ "$(uname -i)" == 'armv7l' ]]; then - ## -I: allow insecure connections while downloading any files - SALT_BOOTSTRAP_OPTS+=( -I ) - _WGET_ARGS+=( --no-check-certificate ) -fi - -echo "Installing saltstack ..." -echo "Option: ${SALT_BOOTSTRAP_OPTS[@]}" -wget ${_WGET_ARGS[@]} -O bootstrap-salt.sh https://bootstrap.saltstack.com -sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" + +## -I: allow insecure connections while downloading any files +is_arm32 && SALT_BOOTSTRAP_OPTS+=( -I ) + +log_info "Installing saltstack ..." +log_debug "Options: ${SALT_BOOTSTRAP_OPTS[@]}" +sh "${BOOTSTRAP_FILE}" ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" chown -R "${SALT_USER}": "${SALT_ROOT_DIR}" # Configure ssh -echo "Configuring ssh ..." +log_info "Configuring ssh ..." sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ssh/ssh_config { echo " UserKnownHostsFile /dev/null" @@ -70,7 +83,7 @@ sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ } >> /etc/ssh/ssh_config # Configure logrotate -echo "Configuring logrotate ..." +log_info "Configuring logrotate ..." # move supervisord.log file to ${SALT_LOGS_DIR}/supervisor/ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;|" /etc/supervisor/supervisord.conf @@ -79,7 +92,7 @@ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;| sed -i "s|^su root syslog$|su root root|" /etc/logrotate.conf # Configure supervisor -echo "Configuring supervisor ..." +log_info "Configuring supervisor ..." # configure supervisord to start salt-master cat > /etc/supervisor/conf.d/salt-master.conf < Removing ${CONTAINER_NAME} ..." + docker container rm --force "${CONTAINER_NAME}" +} + +# trap cleanup EXIT + +# Run test instance +echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..." +docker run --rm --detach --name "${CONTAINER_NAME}" \ + --publish 4505:4505 --publish 4506:4506 \ + --platform "${PLATFORM}" \ + --volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \ + --volume "$(pwd)/tests/gitfs/data/keys":/home/salt/data/keys \ + "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) +echo "container started โœ…" + +# Wait for salt-master bootup +echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." +sleep "${BOOTUP_WAIT_SECONDS}" + +# Check pillars +echo "==> Checking gitfs files ..." +# docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update +# docker exec "${CONTAINER_NAME}" salt-run fileserver.update +FILE_LIST=$(docker exec "${CONTAINER_NAME}" salt-run fileserver.file_list) +echo "${FILE_LIST}" +[[ "${FILE_LIST}" == *test.txt* ]] || ( echo "gitfs files โŒ"; exit 1 ) +echo "gitfs files โœ…" diff --git a/tests/salt-api/test.sh b/tests/salt-api/test.sh index aef77313..4ab13bee 100755 --- a/tests/salt-api/test.sh +++ b/tests/salt-api/test.sh @@ -52,24 +52,24 @@ echo "container started โœ…" echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." sleep "${BOOTUP_WAIT_SECONDS}" - # Test salt-api authentication - echo "==> Getting salt-api token ..." +# Test salt-api authentication +echo "==> Getting salt-api token ..." SALTAPI_TOKEN=$(curl -sSk "${SALTAPI_URL%/}/login" \ - -H "Accept: application/x-yaml" \ - -d username="${SALTAPI_USER}" \ - -d password="${SALTAPI_PASS}" \ - -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) + -H "Accept: application/x-yaml" \ + -d username="${SALTAPI_USER}" \ + -d password="${SALTAPI_PASS}" \ + -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) [ -n "${SALTAPI_TOKEN}" ] || ( echo "salt-api token โŒ"; exit 1 ) echo "salt-api token โœ…" - # Test salt-api command - echo "==> Testing curl command ..." +# Test salt-api command +echo "==> Testing curl command ..." curl -sSk "${SALTAPI_URL}" \ - -H "Accept: application/x-yaml" \ - -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ - -d client=runner \ - -d tgt='*' \ - -d fun=test.stream \ + -H "Accept: application/x-yaml" \ + -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ + -d client=runner \ + -d tgt='*' \ + -d fun=test.stream \ | grep -i 'true' || ( echo "curl command โŒ"; exit 1 ) echo "curl command โœ…" From e6d0737d8536491f712b8d2a39550aa8daa7979a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20A=CC=81lvaro?= Date: Thu, 4 Nov 2021 15:45:29 +0100 Subject: [PATCH 05/11] feat: Change Docker base image to ubuntu:hirsute-20210917 --- CHANGELOG.md | 1 + Dockerfile | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72be1a0c..9c281777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ for the list of changes in SaltStack. - Install `libssh2 1.10.0` from source - Install `libgit2 1.3.0` from source - Install `pygit2 1.7.0` from pip repositories +- Change Docker base image to `ubuntu:hirsute-20210917` **3004** diff --git a/Dockerfile b/Dockerfile index b3bedcb7..3b39c48f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:focal-20211006 +FROM ubuntu:hirsute-20210917 ARG BUILD_DATE ARG VCS_REF diff --git a/README.md b/README.md index 5bb017ad..7ad8e162 100644 --- a/README.md +++ b/README.md @@ -586,7 +586,7 @@ Where `salt-service` is one of: `salt-master` os `salt-api` (if `SALT_API_SERVIC [saltproject_badge]: https://img.shields.io/badge/Salt-v3004-lightgrey.svg?logo=Saltstack [saltproject_release_notes]: https://docs.saltproject.io/en/latest/topics/releases/3004.html "Salt Project Release Notes" -[ubuntu_badge]: https://img.shields.io/badge/ubuntu-focal--20211006-E95420.svg?logo=Ubuntu +[ubuntu_badge]: https://img.shields.io/badge/ubuntu-hirsute--20210917-E95420.svg?logo=Ubuntu [ubuntu_hub_docker]: https://hub.docker.com/_/ubuntu/ "Ubuntu Image" [github_publish_badge]: https://img.shields.io/github/workflow/status/cdalvaro/docker-salt-master/Publish%20Docker%20image?label=build&logo=GitHub&logoColor=%23181717 [github_publish_workflow]: https://github.com/cdalvaro/docker-salt-master/actions?query=workflow%3A%22Publish+Docker+image%22 From 40e8482b03149b91b5abb589a1b5aa859e155703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20A=CC=81lvaro?= Date: Thu, 4 Nov 2021 15:46:28 +0100 Subject: [PATCH 06/11] Upgrade Python to version 3.9 --- CHANGELOG.md | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c281777..b327d7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ for the list of changes in SaltStack. - Install `libgit2 1.3.0` from source - Install `pygit2 1.7.0` from pip repositories - Change Docker base image to `ubuntu:hirsute-20210917` +- Upgrade Python to version `3.9` **3004** diff --git a/Dockerfile b/Dockerfile index 3b39c48f..28c2b32f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG VCS_REF # https://github.com/saltstack/salt/releases ENV SALT_VERSION="3004" \ - PYTHON_VERSION="3.8" + PYTHON_VERSION="3.9" ENV IMAGE_VERSION="${SALT_VERSION}_1" From f15d4473a4749eea23c68c4dd3edc67e059b645b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 4 Nov 2021 19:09:56 +0100 Subject: [PATCH 07/11] test(ci): Add common.sh utils --- tests/lib/common.sh | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/lib/common.sh diff --git a/tests/lib/common.sh b/tests/lib/common.sh new file mode 100644 index 00000000..4dc7e7fd --- /dev/null +++ b/tests/lib/common.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: IMAGE_NAME +# DESCRIPTION: The name and tag of the Docker image. Default: 'cdalvaro/docker-salt-master:latest'. +#---------------------------------------------------------------------------------------------------------------------- +export IMAGE_NAME=${IMAGE_NAME:-'cdalvaro/docker-salt-master:latest'} + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: CONTAINER_NAME +# DESCRIPTION: The name of the container. Default: 'salt-master'. +#---------------------------------------------------------------------------------------------------------------------- +export CONTAINER_NAME=salt-master + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: PLATFORM +# DESCRIPTION: The platform to run the tests on. Default: the current platform. +#---------------------------------------------------------------------------------------------------------------------- +export PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: BOOTUP_WAIT_SECONDS +# DESCRIPTION: The number of seconds to wait for the container to boot up. Default: 60. +#---------------------------------------------------------------------------------------------------------------------- +export BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: cleanup +# DESCRIPTION: Clean up tasks. +#---------------------------------------------------------------------------------------------------------------------- +function cleanup() +{ + echo "==> Removing ${CONTAINER_NAME} ..." + docker container rm --force "${CONTAINER_NAME}" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: master_log +# DESCRIPTION: Print salt-master log. +#---------------------------------------------------------------------------------------------------------------------- +function master_log() +{ + docker exec "${CONTAINER_NAME}" cat data/logs/salt/master +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: wait_container +# DESCRIPTION: Wait for the container to boot up. +#---------------------------------------------------------------------------------------------------------------------- +function start_container_and_wait() +{ + # shellcheck disable=SC2206 + local DOCKER_ARGS=( $@ ) + + docker run --rm --detach --name "${CONTAINER_NAME}" \ + --publish 4505:4505 --publish 4506:4506 \ + --platform "${PLATFORM}" ${DOCKER_ARGS[@]} \ + "${IMAGE_NAME}" || return 1 + + echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." + sleep "${BOOTUP_WAIT_SECONDS}" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: ok +# DESCRIPTION: Print a successfull message. +#---------------------------------------------------------------------------------------------------------------------- +function ok() +{ + echo "โœ… $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: error +# DESCRIPTION: Print an error message, show the salt-master log and exit with code 1. +#---------------------------------------------------------------------------------------------------------------------- +function error() +{ + echo "๐Ÿ”ฅ $*" + master_log + return 1 +} From 73c617bb3d12d340aebc2ad530660d9848fa7a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 4 Nov 2021 19:10:34 +0100 Subject: [PATCH 08/11] fix(test): Fix GitFS tests --- .github/workflows/build-and-test.yml | 4 +-- tests/gitfs/test.sh | 40 +++++++++++++--------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7d74b685..8e841108 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -124,8 +124,8 @@ jobs: GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs run: | mkdir -p "${GITFS_KEYS_DIR}" - echo ${{ secrets.TESTS_REPO_PRIVATE_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$ - echo ${{ secrets.TESTS_REPO_PUBLIC_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$ + echo "${{ secrets.TESTS_REPO_PRIVATE_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$ + echo "${{ secrets.TESTS_REPO_PUBLIC_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$ tests/gitfs/test.sh - name: Cleanup diff --git a/tests/gitfs/test.sh b/tests/gitfs/test.sh index bd49a740..e2f6592b 100755 --- a/tests/gitfs/test.sh +++ b/tests/gitfs/test.sh @@ -3,39 +3,35 @@ set -e [ "${DEBUG}" == true ] && set -vx -echo "๐Ÿงช Running gitfs tests ..." +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -IMAGE_NAME=${IMAGE_NAME:-cdalvaro/docker-salt-master} -CONTAINER_NAME=salt_master -PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} -BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" -function cleanup { - echo "==> Removing ${CONTAINER_NAME} ..." - docker container rm --force "${CONTAINER_NAME}" -} +echo "๐Ÿงช Running gitfs tests ..." -# trap cleanup EXIT +trap cleanup EXIT # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..." -docker run --rm --detach --name "${CONTAINER_NAME}" \ - --publish 4505:4505 --publish 4506:4506 \ - --platform "${PLATFORM}" \ +start_container_and_wait \ --volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \ --volume "$(pwd)/tests/gitfs/data/keys":/home/salt/data/keys \ - "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) -echo "container started โœ…" +|| error "container started" +ok "container started" -# Wait for salt-master bootup -echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." -sleep "${BOOTUP_WAIT_SECONDS}" +# Update repositories +echo "==> Updating gitfs repositories ..." +docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update +UPDATE_REPOS="$(docker exec "${CONTAINER_NAME}" salt-run fileserver.update)" +echo "${UPDATE_REPOS}" | grep -qi 'true' || error "update gitfs" +ok "update gitfs" # Check pillars echo "==> Checking gitfs files ..." -# docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update -# docker exec "${CONTAINER_NAME}" salt-run fileserver.update FILE_LIST=$(docker exec "${CONTAINER_NAME}" salt-run fileserver.file_list) echo "${FILE_LIST}" -[[ "${FILE_LIST}" == *test.txt* ]] || ( echo "gitfs files โŒ"; exit 1 ) -echo "gitfs files โœ…" +[[ "${FILE_LIST}" == *test.txt* ]] || error "gitfs files" +ok "gitfs files" From c85cbbf000aeec2ad2fe02a6577563a02a2ae7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 4 Nov 2021 19:48:55 +0100 Subject: [PATCH 09/11] ref(tests): Use common.sh on tests --- tests/basic/test.sh | 35 +++++++++++-------------------- tests/gitfs/test.sh | 20 ++++++++++++------ tests/lib/common.sh | 29 +++++++++++++++++++++++++- tests/salt-api/test.sh | 47 +++++++++++++++++------------------------- 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/tests/basic/test.sh b/tests/basic/test.sh index 50011638..05c4bd82 100755 --- a/tests/basic/test.sh +++ b/tests/basic/test.sh @@ -5,37 +5,26 @@ set -e echo "๐Ÿงช Running basic tests ..." -IMAGE_NAME=${IMAGE_NAME:-cdalvaro/docker-salt-master} -CONTAINER_NAME=salt_master -PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} -BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} - -function cleanup { - echo "==> Removing ${CONTAINER_NAME} ..." - docker container rm --force "${CONTAINER_NAME}" -} +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" trap cleanup EXIT # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) ..." -docker run --rm --detach --name "${CONTAINER_NAME}" \ - --publish 4505:4505 --publish 4506:4506 \ - --platform "${PLATFORM}" \ - "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) -echo "container started โœ…" - -# Wait for salt-master bootup -echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." -sleep "${BOOTUP_WAIT_SECONDS}" +start_container_and_wait || error "container started" +ok "container started" # Check salt version echo "==> Checking salt version ..." -docker exec "${CONTAINER_NAME}" salt --versions -[[ "$(docker exec ${CONTAINER_NAME} salt --version)" == "salt $(cat VERSION)" ]] || ( echo "salt version โŒ"; exit 1 ) -echo "salt version โœ…" +docker-exec salt --versions +[[ "$(docker-exec salt --version)" == "salt $(cat VERSION)" ]] || error "salt version" +ok "salt version" # Test image calling healthcheck echo "==> Executing healthcheck ..." -docker exec "${CONTAINER_NAME}" /usr/local/sbin/healthcheck | grep -i 'true' || ( echo "healthcheck โŒ"; exit 1 ) -echo "healthcheck โœ…" +docker-exec /usr/local/sbin/healthcheck | grep -i 'true' || error "healthcheck" +ok "healthcheck" diff --git a/tests/gitfs/test.sh b/tests/gitfs/test.sh index e2f6592b..3600fc67 100755 --- a/tests/gitfs/test.sh +++ b/tests/gitfs/test.sh @@ -3,35 +3,43 @@ set -e [ "${DEBUG}" == true ] && set -vx +echo "๐Ÿงช Running gitfs tests ..." + # https://stackoverflow.com/a/4774063/3398062 SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # shellcheck source=assets/build/functions.sh COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" source "${COMMON_FILE}" +trap cleanup EXIT -echo "๐Ÿงช Running gitfs tests ..." +export GITFS_KEYS_DIR=${GITFS_KEYS_DIR:-tests/gitfs/data/keys/gitfs} -trap cleanup EXIT +# Check gitfs keys are present +echo "==> Checking gitfs keys are present ..." +if [[ ! -f "${GITFS_KEYS_DIR}/gitfs_ssh" || ! -f "${GITFS_KEYS_DIR}/gitfs_ssh.pub" ]]; then + error "gitfs keys not found at ${GITFS_KEYS_DIR}" +fi +ok "gitfs keys" # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..." start_container_and_wait \ --volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \ - --volume "$(pwd)/tests/gitfs/data/keys":/home/salt/data/keys \ + --volume "$(pwd)/${GITFS_KEYS_DIR%%/gitfs}":/home/salt/data/keys \ || error "container started" ok "container started" # Update repositories echo "==> Updating gitfs repositories ..." -docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update -UPDATE_REPOS="$(docker exec "${CONTAINER_NAME}" salt-run fileserver.update)" +salt-run cache.clear_git_lock gitfs type=update +UPDATE_REPOS="$( salt-run fileserver.update )" echo "${UPDATE_REPOS}" | grep -qi 'true' || error "update gitfs" ok "update gitfs" # Check pillars echo "==> Checking gitfs files ..." -FILE_LIST=$(docker exec "${CONTAINER_NAME}" salt-run fileserver.file_list) +FILE_LIST=$( salt-run fileserver.file_list ) echo "${FILE_LIST}" [[ "${FILE_LIST}" == *test.txt* ]] || error "gitfs files" ok "gitfs files" diff --git a/tests/lib/common.sh b/tests/lib/common.sh index 4dc7e7fd..0922eb3e 100644 --- a/tests/lib/common.sh +++ b/tests/lib/common.sh @@ -34,13 +34,40 @@ function cleanup() docker container rm --force "${CONTAINER_NAME}" } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: docker-exec +# DESCRIPTION: Execute the given command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function docker-exec() +{ + docker exec "${CONTAINER_NAME}" "$@" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: salt-run +# DESCRIPTION: Execute the salt-run command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function salt-run() +{ + docker-exec salt-run "$@" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: salt-call +# DESCRIPTION: Execute the salt-call command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function salt-call() +{ + docker-exec salt-call "$@" +} + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: master_log # DESCRIPTION: Print salt-master log. #---------------------------------------------------------------------------------------------------------------------- function master_log() { - docker exec "${CONTAINER_NAME}" cat data/logs/salt/master + docker-exec cat data/logs/salt/master } #--- FUNCTION ------------------------------------------------------------------------------------------------------- diff --git a/tests/salt-api/test.sh b/tests/salt-api/test.sh index 4ab13bee..5f661051 100755 --- a/tests/salt-api/test.sh +++ b/tests/salt-api/test.sh @@ -5,10 +5,13 @@ set -e echo "๐Ÿงช Running salt-api tests ..." -IMAGE_NAME=${IMAGE_NAME:-cdalvaro/docker-salt-master} -CONTAINER_NAME=salt_master -PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} -BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" +trap cleanup EXIT export SALTAPI_URL="https://localhost:8000/" export SALTAPI_USER=salt_api @@ -16,13 +19,6 @@ export SALTAPI_PASS=4wesome-Pass0rd export SALTAPI_EAUTH=pam export SALTAPI_TMP_DIR=${SALTAPI_TMP_DIR:-/tmp/salt-api} -function cleanup { - echo "==> Removing ${CONTAINER_NAME} ..." - docker container rm --force "${CONTAINER_NAME}" -} - -trap cleanup EXIT - # Create configuration files echo "==> Creating salt-api configuration file ..." mkdir -p "${SALTAPI_TMP_DIR}/config/" @@ -39,18 +35,13 @@ echo "salt-api config created โœ…" # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) with salt-api config ..." -docker run --rm --detach --name "${CONTAINER_NAME}" \ - --publish 4505:4505 --publish 4506:4506 --publish 8000:8000 \ +start_container_and_wait \ + --publish 8000:8000 \ --env SALT_API_SERVICE_ENABLED=true \ --env SALT_API_USER_PASS="${SALTAPI_PASS}" \ - --platform "${PLATFORM}" \ --volume "${SALTAPI_TMP_DIR}/config":/home/salt/data/config:ro \ - "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) -echo "container started โœ…" - -# Wait for salt-master bootup -echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." -sleep "${BOOTUP_WAIT_SECONDS}" +|| error "container started" +ok "container started" # Test salt-api authentication echo "==> Getting salt-api token ..." @@ -59,8 +50,8 @@ SALTAPI_TOKEN=$(curl -sSk "${SALTAPI_URL%/}/login" \ -d username="${SALTAPI_USER}" \ -d password="${SALTAPI_PASS}" \ -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) -[ -n "${SALTAPI_TOKEN}" ] || ( echo "salt-api token โŒ"; exit 1 ) -echo "salt-api token โœ…" +[ -n "${SALTAPI_TOKEN}" ] || error "salt-api token" +ok "salt-api token" # Test salt-api command echo "==> Testing curl command ..." @@ -70,15 +61,15 @@ curl -sSk "${SALTAPI_URL}" \ -d client=runner \ -d tgt='*' \ -d fun=test.stream \ -| grep -i 'true' || ( echo "curl command โŒ"; exit 1 ) -echo "curl command โœ…" +| grep -i 'true' || error "curl command" +ok "curl command" # Install salt-pepper echo "==> Installing salt-pepper ..." -pip3 install salt-pepper || ( echo "pepper installed โŒ"; exit 1 ) -echo "pepper installed โœ…" +pip3 install salt-pepper || error "pepper installed" +ok "pepper installed" # Test salt-pepper echo "==> Testing salt-pepper ..." -pepper -vvv --debug-http --ignore-ssl-errors --client runner test.stream|| ( echo "pepper test.stream โŒ"; exit 1 ) -echo "pepper test.stream โœ…" +pepper --client runner test.stream || error "pepper test.stream" +ok "pepper test.stream" From 8d3f0317e415b1a629f9d19102e6b21b4c92ddd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 4 Nov 2021 21:42:12 +0100 Subject: [PATCH 10/11] fix(test): Fix gitfs keys generation --- .github/workflows/build-and-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 8e841108..8f223912 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -124,8 +124,10 @@ jobs: GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs run: | mkdir -p "${GITFS_KEYS_DIR}" - echo "${{ secrets.TESTS_REPO_PRIVATE_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$ - echo "${{ secrets.TESTS_REPO_PUBLIC_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$ + echo "${{ secrets.TESTS_REPO_PRIVATE_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh + chmod 600 "${GITFS_KEYS_DIR}"/gitfs_ssh + echo "${{ secrets.TESTS_REPO_PUBLIC_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub + chmod 644 "${GITFS_KEYS_DIR}"/gitfs_ssh.pub tests/gitfs/test.sh - name: Cleanup From 452c1f56577a1c52c5235f711190f2a13338e3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Fri, 5 Nov 2021 14:24:42 +0100 Subject: [PATCH 11/11] doc: Update functions documentation --- assets/build/functions.sh | 44 +++++++-------------- assets/build/install.sh | 5 +-- assets/runtime/functions.sh | 77 +++++++++++++++++++++++++++++-------- 3 files changed, 76 insertions(+), 50 deletions(-) diff --git a/assets/build/functions.sh b/assets/build/functions.sh index 8254e0f9..5d71e639 100755 --- a/assets/build/functions.sh +++ b/assets/build/functions.sh @@ -2,36 +2,13 @@ set -e -#--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __detect_color_support -# DESCRIPTION: Try to detect color support. -#---------------------------------------------------------------------------------------------------------------------- -_COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} -__detect_color_support() { - # shellcheck disable=SC2181 - if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then - RC='\033[1;31m' - GC='\033[1;32m' - BC='\033[1;34m' - YC='\033[1;33m' - EC='\033[0m' - else - RC="" - GC="" - BC="" - YC="" - EC="" - fi -} -__detect_color_support - #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: log_debug # DESCRIPTION: Echo debug information to stdout. #---------------------------------------------------------------------------------------------------------------------- function log_debug() { if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then - echo -e "${BC} * DEBUG${EC}: $*" + echo " * DEBUG: $*" fi } @@ -40,7 +17,7 @@ function log_debug() { # DESCRIPTION: Echo information to stdout. #---------------------------------------------------------------------------------------------------------------------- function log_info() { - echo -e "${GC} * INFO${EC}: $*" + echo " * INFO: $*" } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -48,7 +25,7 @@ function log_info() { # DESCRIPTION: Echo warning information to stdout. #---------------------------------------------------------------------------------------------------------------------- function log_warn() { - echo -e "${YC} * WARN${EC}: $*" + echo " * WARN: $*" } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -57,12 +34,12 @@ function log_warn() { #---------------------------------------------------------------------------------------------------------------------- function log_error() { - (>&2 echo -e "${RC} * ERROR${EC}: $*") + (>&2 echo " * ERROR: $*") } #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: exec_as_salt -# DESCRIPTION: Execute the pass command as the `salt` user. +# DESCRIPTION: Execute the pass command as the `SALT_USER` user. #---------------------------------------------------------------------------------------------------------------------- function exec_as_salt() { @@ -100,9 +77,13 @@ function is_arm() is_arm32 || is_arm64 } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_pkgs +# DESCRIPTION: Install packages using apt-get install. +#---------------------------------------------------------------------------------------------------------------------- function install_pkgs() { - apt-get install --no-install-recommends -y $@ + apt-get install --no-install-recommends --yes $@ } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -213,7 +194,10 @@ function install_libssh2() rm -rf "${WORK_DIR}" } -# Install libgit2 library +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_libgit2 +# DESCRIPTION: Install libgit2 library. +#---------------------------------------------------------------------------------------------------------------------- function install_libgit2() { local LIBGIT2_VERSION=1.3.0 diff --git a/assets/build/install.sh b/assets/build/install.sh index c47e71a5..b68385e8 100755 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -19,8 +19,7 @@ BUILD_DEPENDENCIES=( ) apt-get update -apt-get install --no-install-recommends -y \ - "${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}" +install_pkgs "${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}" # Create salt user log_info "Creating ${SALT_USER} user ..." @@ -33,7 +32,7 @@ EOF # Install python3 packages log_info "Installing python3 packages ..." -apt-get install --yes --quiet --no-install-recommends \ +install_pkgs --quiet \ python3-mako python3-pycryptodome python3-cherrypy3 \ python3-git python3-requests python3-redis python3-gnupg \ python3-mysqldb python3-dateutil python3-libnacl python3-openssl diff --git a/assets/runtime/functions.sh b/assets/runtime/functions.sh index 69b75acf..bb4d428d 100755 --- a/assets/runtime/functions.sh +++ b/assets/runtime/functions.sh @@ -9,7 +9,10 @@ source "${ENV_DEFAULTS_FILE}" # cdalvaro managed block string SELF_MANAGED_BLOCK_STRING="## cdalvaro managed block" -# Execute a command as SALT_USER +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: exec_as_salt +# DESCRIPTION: Execute the pass command as the `SALT_USER` user. +#---------------------------------------------------------------------------------------------------------------------- function exec_as_salt() { if [[ $(whoami) == "${SALT_USER}" ]]; then @@ -19,13 +22,19 @@ function exec_as_salt() fi } -# Log error +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_error +# DESCRIPTION: Echo errors to stderr. +#---------------------------------------------------------------------------------------------------------------------- function log_error() { - (>&2 echo "ERROR: $*") + (>&2 echo " * ERROR: $*") } -# Map salt user with host user +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: map_uidgid +# DESCRIPTION: Map salt user with host user. +#---------------------------------------------------------------------------------------------------------------------- function map_uidgid() { USERMAP_ORIG_UID=$(id -u "${SALT_USER}") @@ -48,9 +57,13 @@ function map_uidgid() fi } -# This function replaces placeholders with values -# $1: file with placeholders to replace -# $x: placeholders to replace +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: update_template +# DESCRIPTION: Replace placeholders with values. +# ARGUMENTS: +# - 1: Template file with placeholders to replace +# - @: Placeholder values +#---------------------------------------------------------------------------------------------------------------------- function update_template() { local FILE=${1?missing argument} @@ -77,7 +90,10 @@ function update_template() rm -f "${tmp_file}" } -# This function configures containers timezone +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_timezone +# DESCRIPTION: Configure containers timezone. +#---------------------------------------------------------------------------------------------------------------------- function configure_timezone() { echo "Configuring container timezone ..." @@ -97,7 +113,10 @@ function configure_timezone() fi } -# This function generates a master_sign key pair and its signature +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: gen_signed_keys +# DESCRIPTION: Generate a master_sign key pair and its signature. +#---------------------------------------------------------------------------------------------------------------------- function gen_signed_keys() { local key_name=${1:-master} @@ -111,7 +130,10 @@ function gen_signed_keys() echo -n "${GENERATED_KEYS_DIR}" } -# This function repairs keys permissions and creates keys if neaded +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: setup_salt_keys +# DESCRIPTION: Repair keys permissions and creates keys if neaded. +#---------------------------------------------------------------------------------------------------------------------- function setup_salt_keys() { echo "Setting up salt keys ..." @@ -138,7 +160,10 @@ function setup_salt_keys() find "${SALT_HOME}" -path "${SALT_KEYS_DIR}/*" -print0 | xargs -0 chown -h "${SALT_USER}": } -# This function configures ssh keys +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: setup_ssh_keys +# DESCRIPTION: Configure ssh keys. +#---------------------------------------------------------------------------------------------------------------------- function setup_ssh_keys() { echo "Configuring ssh ..." @@ -156,7 +181,10 @@ function setup_ssh_keys() fi } -# This function cofigures master service +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_salt_master +# DESCRIPTION: Configure master service. +#---------------------------------------------------------------------------------------------------------------------- function configure_salt_master() { echo "Configuring salt-master service ..." @@ -183,7 +211,10 @@ function configure_salt_master() SALT_MASTER_USE_PUBKEY_SIGNATURE } -# This function configures salt-api if service is set to be enabled +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_salt_api +# DESCRIPTION: Configure salt-api if service is set to be enabled. +#---------------------------------------------------------------------------------------------------------------------- function configure_salt_api() { [[ ${SALT_API_SERVICE_ENABLED} == true ]] || return 0 @@ -246,7 +277,10 @@ EOF } -# This function configures salt-formulas +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_salt_formulas +# DESCRIPTION: Configure salt-formulas. +#---------------------------------------------------------------------------------------------------------------------- function configure_salt_formulas() { echo "Configuring 3rd-party salt-formulas ..." @@ -266,7 +300,10 @@ function configure_salt_formulas() rm "${tmp_file}" } -# Initializes main directories +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: initialize_datadir +# DESCRIPTION: Initialize main directories. +#---------------------------------------------------------------------------------------------------------------------- function initialize_datadir() { echo "Configuring directories ..." @@ -328,7 +365,10 @@ function initialize_datadir() chown -R "${SALT_USER}": "${SALT_LOGS_DIR}/salt" } -# Configures logrotate +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_logrotate +# DESCRIPTION: Configure logrotate. +#---------------------------------------------------------------------------------------------------------------------- function configure_logrotate() { echo "Configuring logrotate ..." @@ -391,7 +431,10 @@ EOF } -# Initializes the system +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: initialize_system +# DESCRIPTION: Initialize the system. +#---------------------------------------------------------------------------------------------------------------------- function initialize_system() { map_uidgid