From f84d818b0e1931739bb99fe1630781b937baa49a Mon Sep 17 00:00:00 2001 From: mrekucci Date: Fri, 12 Jul 2024 12:54:41 +0200 Subject: [PATCH] feat: add auth_token as a new type of secret artifact --- infrastructure/nomad/playbooks/deploy.yml | 265 ++++++++++-------- .../nomad/playbooks/variables/profiles.yml | 2 + 2 files changed, 146 insertions(+), 121 deletions(-) diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index 6770aec74..27d1403c0 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -188,29 +188,28 @@ - name: Build Artifacts Async ansible.builtin.shell: | + SECRETS="{{ dist_dir }}/{{ item.name }}_secrets.json" SIGNERS="{{ dist_dir }}/signers.txt" - SECRETS="{{ dist_dir }}/secrets.json" INVENTORY="{{ dist_dir }}/artifacts.txt" ALLOCATIONS="{{ dist_dir }}/allocations.txt" DESTINATION_DIR="{{ dist_dir }}/{{ item.name }}" && mkdir -p ${DESTINATION_DIR} - + + [ ! -f "${SECRETS}" ] && echo '{}' > "${SECRETS}" [ ! -f "${SIGNERS}" ] && touch "${SIGNERS}" - [ ! -f "${SECRETS}" ] && echo '{"version":"{{ environments[env].version }}"}' > "${SECRETS}" [ ! -f "${INVENTORY}" ] && touch "${INVENTORY}" [ ! -f "${ALLOCATIONS}" ] && touch "${ALLOCATIONS}" - exec 190>>"${SIGNERS}" - exec 191<>"${SECRETS}" - exec 192<>"${INVENTORY}" - exec 193>>"${ALLOCATIONS}" + exec 192>>"${SIGNERS}" + exec 193<>"${INVENTORY}" + exec 194>>"${ALLOCATIONS}" {% for artifact in item.artifacts | default([]) %} {% if artifact.type is defined and artifact.type == 'binary' %} - flock -v -x 192 + flock -v -x 193 if ! grep -qxF "{{ artifact.path }}" "${INVENTORY}"; then echo "{{ artifact.path }}" >> "${INVENTORY}" - flock -v -u 192 + flock -v -u 193 set -x cp ./{{ artifact.path }}/.goreleaser.yml ./{{ artifact.path }}/.goreleaser.tmp.yml if [ -n "${ARTIFACTS_GOOS}" ]; then @@ -224,140 +223,147 @@ if [ $? -ne 0 ]; then exit 1; fi rm ./{{ artifact.path }}/.goreleaser.tmp.yml fi - flock -v -u 192 + flock -v -u 193 {% elif artifact.type is defined and artifact.type == 'archive' %} ARTIFACT="${DESTINATION_DIR}/{{ artifact.name | default(item.name) }}_{{ environments[env].version }}.tar.gz" if [ ! -f "${ARTIFACT}" ]; then - flock -v -x 192 + flock -v -x 193 echo "${ARTIFACT}" >> "${INVENTORY}" - flock -v -u 192 + flock -v -u 193 tar -czvf "${ARTIFACT}" ./{{ artifact.path }} if [ $? -ne 0 ]; then exit 1; fi fi {% endif %} - {% if artifact.keystore is defined %} - case "{{ environments[env].secrets }}" in - "generate") - PASSPHRASE="{{ lookup('password', '/dev/null', length=1024, chars=['ascii_letters', 'digits']) }}" - - RESULT=$( - {{ keystore_generator.stdout }} generate \ - --keystore-dir="${DESTINATION_DIR}" \ - --passphrase="${PASSPHRASE}" \ - --log-fmt="json" - ) - if [ $? -ne 0 ]; then exit 1; fi - ARTIFACT_PATH="$(echo ${RESULT} | jq -e -r '.path // empty' 2>/dev/null)" + case "{{ environments[env].secrets }}" in + "generate") + {% if artifact.keystore is defined %} + PASSPHRASE="{{ lookup('password', '/dev/null', length=1024, chars=['ascii_letters', 'digits']) }}" + + RESULT=$( + {{ keystore_generator.stdout }} generate \ + --keystore-dir="${DESTINATION_DIR}" \ + --passphrase="${PASSPHRASE}" \ + --log-fmt="json" + ) + if [ $? -ne 0 ]; then exit 1; fi + ARTIFACT_PATH="$(echo ${RESULT} | jq -e -r '.path // empty' 2>/dev/null)" + + cat "${SECRETS}" | jq \ + --arg keystore "$(cat ${ARTIFACT_PATH} | jq -c .)" \ + --arg keystore_filename "$(basename ${ARTIFACT_PATH})" \ + --arg keystore_password "${PASSPHRASE}" \ + '. + { + "{{ artifact.keystore.name }}": $keystore, + "{{ artifact.keystore.name }}_filename": $keystore_filename, + "{{ artifact.keystore.name }}_password": $keystore_password + }' \ + > "${SECRETS}.tmp" && mv "${SECRETS}.tmp" "${SECRETS}" + if [ $? -ne 0 ]; then + echo "Error: Failed to write keystore to secrets for {{ item.name }}." + exit 1; + fi + + ADDRESS="$(cat ${ARTIFACT_PATH} | jq -r '.address')" + {% if artifact.keystore.allocation | default(false) %} + flock -v -x 194 + echo "${ADDRESS}" >> "${ALLOCATIONS}" + flock -v -u 194 + {% endif %} + {% if item.env is defined and item.env.type | default('') == 'signer' %} flock -v -x 192 - echo "${ARTIFACT_PATH}" >> "${INVENTORY}" + echo "${ADDRESS}" >> "${SIGNERS}" flock -v -u 192 + {% endif %} + {% endif %} - flock -v -x 191 - cat "${SECRETS}" | jq \ - --arg keystore "$(cat ${ARTIFACT_PATH} | jq -c .)" \ - --arg keystore_filename "$(basename ${ARTIFACT_PATH})" \ - --arg keystore_password "${PASSPHRASE}" \ - '. + { - "{{ artifact.keystore.name }}": $keystore, - "{{ artifact.keystore.name }}_filename": $keystore_filename, - "{{ artifact.keystore.name }}_password": $keystore_password - }' \ - > "${SECRETS}" - if [ $? -ne 0 ]; then exit 1; fi - flock -v -u 191 - - ADDRESS="$(cat ${ARTIFACT_PATH} | jq -r '.address')" - {% if artifact.keystore.allocation | default(false) %} - flock -v -x 193 - echo "${ADDRESS}" >> "${ALLOCATIONS}" - flock -v -u 193 - {% endif %} - {% if item.env is defined and item.env.type | default('') == 'signer' %} - flock -v -x 190 - echo "${ADDRESS}" >> "${SIGNERS}" - flock -v -u 190 - {% endif %} - ;; - "fetch") - RESPONSE=$(curl \ - --silent \ - --insecure \ - --header "X-Vault-Token: {{ vault_init.json.root_token }}" \ - {{ vault_address }}/v1/{{ vault_kv_engine_path }}/data/{{ vault_secret_path }} - ) - if [ -z "${RESPONSE}" ]; then - echo "Error: No secrets found." - exit 1 - fi + {% if artifact.boot_key | default(false) %} + bootnode -genkey "${DESTINATION_DIR}/boot.key" + if [ $? -ne 0 ]; then exit 1; fi - KEYSTORE=$(echo "${RESPONSE}" | jq -r '.data.data.{{ artifact.keystore.name }}') - if [ -z "${KEYSTORE}" ]; then - echo "Error: No keystore found for {{ artifact.keystore.name }}." - exit 1 - fi - PASSPHRASE=$(echo "${RESPONSE}" | jq -r '.data.data.{{ artifact.keystore.name }}_password') - if [ -z "${PASSPHRASE}" ]; then - echo "Error: No passphrase found for {{ artifact.keystore.name }}." + cat "${SECRETS}" | jq \ + --arg item_name "{{ (item.name | regex_replace('^mev-commit-', '') | replace('-', '_')) }}" \ + --arg boot_key "$(cat ${DESTINATION_DIR}/boot.key)" \ + --arg boot_key_address "$(bootnode -nodekey ${DESTINATION_DIR}/boot.key -writeaddress)" \ + '. + { + ($item_name + "_boot_key"): $boot_key, + ($item_name + "_boot_key_address"): $boot_key_address + }' \ + > "${SECRETS}.tmp" && mv "${SECRETS}.tmp" "${SECRETS}" + if [ $? -ne 0 ]; then + echo "Error: Failed to write boot key to secrets for {{ item.name }}." + exit 1; + fi + {% endif %} + + {% if artifact.auth_token | default(false) %} + set -x + cat "${SECRETS}" | jq \ + --arg item_name "{{ artifact.auth_token.name }}" \ + --arg auth_token "{{ lookup('password', '/dev/null', length=128, chars=['ascii_letters', 'digits']) }}" \ + '. + { + ($item_name): $auth_token + }' \ + > "${SECRETS}.tmp" && mv "${SECRETS}.tmp" "${SECRETS}" + if [ $? -ne 0 ]; then + echo "Error: Failed to write auth token to secrets for {{ item.name }}." exit 1 fi + set +x + {% endif %} + ;; + "fetch") + RESPONSE=$(curl \ + --silent \ + --insecure \ + --header "X-Vault-Token: {{ vault_init.json.root_token }}" \ + {{ vault_address }}/v1/{{ vault_kv_engine_path }}/data/{{ vault_secret_path }} + ) + if [ $? -ne 0 ]; then + echo "Error: Failed to fetch secrets." + exit 1 + fi + if [ -z "${RESPONSE}" ]; then + echo "Error: No secrets found." + exit 1 + fi - flock -v -x 191 - cat "${SECRETS}" | jq \ - --arg keystore "$(echo $KEYSTORE | jq -c .)" \ - --arg keystore_filename "$(basename ${KEYSTORE})" \ - --arg keystore_password "${PASSPHRASE}" \ - '. + { - "{{ artifact.keystore.name }}": $keystore, - "{{ artifact.keystore.name }}_filename": $keystore_filename, - "{{ artifact.keystore.name }}_password": $keystore_password - }' \ - > "${SECRETS}" - if [ $? -ne 0 ]; then exit 1; fi - flock -v -u 191 - - ADDRESS=$(echo "${KEYSTORE}" | jq -r '.address') - {% if artifact.keystore.allocation | default(false) %} - flock -v -x 193 - echo "${ADDRESS}" >> "${ALLOCATIONS}" - flock -v -u 193 - {% endif %} - {% if item.env is defined and item.env.type | default('') == 'signer' %} - flock -v -x 190 - echo "${ADDRESS}" >> "${SIGNERS}" - flock -v -u 190 - {% endif %} - ;; - *) - echo "Error: Unknown secrets type: {{ environments[env].secrets }}." + {% if artifact.keystore is defined %} + KEYSTORE=$(echo "${RESPONSE}" | jq -r '.data.data.{{ artifact.keystore.name }}') + if [ -z "${KEYSTORE}" ]; then + echo "Error: No keystore found for {{ artifact.keystore.name }}." exit 1 - ;; - esac - {% endif %} - {% if artifact.boot_key | default(false) and environments[env].secrets == 'generate' %} - bootnode -genkey "${DESTINATION_DIR}/boot.key" - if [ $? -ne 0 ]; then exit 1; fi - - flock -v -x 191 - cat "${SECRETS}" | jq \ - --arg item_name "{{ (item.name | regex_replace('^mev-commit-', '') | replace('-', '_')) }}" \ - --arg boot_key "$(cat ${DESTINATION_DIR}/boot.key)" \ - --arg boot_key_address "$(bootnode -nodekey ${DESTINATION_DIR}/boot.key -writeaddress)" \ - '. + { - ($item_name + "_boot_key"): $boot_key, - ($item_name + "_boot_key_address"): $boot_key_address - }' \ - > "${SECRETS}" - if [ $? -ne 0 ]; then exit 1; fi - flock -v -u 191 + fi + PASSPHRASE=$(echo "${RESPONSE}" | jq -r '.data.data.{{ artifact.keystore.name }}_password') + if [ -z "${PASSPHRASE}" ]; then + echo "Error: No passphrase found for {{ artifact.keystore.name }}." + exit 1 + fi + + ADDRESS=$(echo "${KEYSTORE}" | jq -r '.address') + {% if artifact.keystore.allocation | default(false) %} + flock -v -x 194 + echo "${ADDRESS}" >> "${ALLOCATIONS}" + flock -v -u 194 + {% endif %} + {% if item.env is defined and item.env.type | default('') == 'signer' %} + flock -v -x 192 + echo "${ADDRESS}" >> "${SIGNERS}" + flock -v -u 192 + {% endif %} + {% endif %} + ;; + *) + echo "Error: Unknown secrets type: {{ environments[env].secrets }}." + exit 1 + ;; + esac - {% endif %} {% endfor %} - exec 190>&- - exec 191>&- exec 192>&- exec 193>&- + exec 194>&- environment: ARTIFACTS_GOOS: "{{ lookup('env', 'ARTIFACTS_GOOS') }}" ARTIFACTS_GOARCH: "{{ lookup('env', 'ARTIFACTS_GOARCH') }}" @@ -390,6 +396,23 @@ run_once: true when: build_artifacts + - name: Assemble Secrets File + ansible.builtin.shell: | + SECRETS=$(ls {{ dist_dir }}/*_secrets.json 2>/dev/null) + if [ -z "${SECRETS}" ]; then + echo "Error: No secrets found." + exit 1 + fi + jq -s ' + reduce .[] as $i ({}; . * $i) + | . + {"version":"{{ environments[env].version }}"} + ' ${SECRETS} > "{{ dist_dir }}/secrets.json" + args: + executable: bash + delegate_to: localhost + run_once: true + when: build_artifacts and environments[env].secrets == 'generate' + - name: Assemble Genesis File ansible.builtin.shell: | ALLOCATIONS="$(cat {{ dist_dir }}/allocations.txt 2>/dev/null)" diff --git a/infrastructure/nomad/playbooks/variables/profiles.yml b/infrastructure/nomad/playbooks/variables/profiles.yml index ece0ee884..bec42ecaf 100644 --- a/infrastructure/nomad/playbooks/variables/profiles.yml +++ b/infrastructure/nomad/playbooks/variables/profiles.yml @@ -452,6 +452,8 @@ jobs: template: mev-commit-oracle.nomad.j2 artifacts: - *oracle_artifact + - auth_token: + name: oracle_register_provider_api_auth_token - keystore: name: oracle_keystore allocation: true