From 46ae1ee387ec9ec8ace27e7f3693e3d2df06cc74 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Fri, 5 Jul 2024 10:42:02 +1000 Subject: [PATCH] UID2-3331 Update ami from al2 to al2023 (#641) * Update AL source * Remove unnecessary arguments * Update installing aws-nitro-enclaves-cli * Use dnf instead of yum * Remove the step for installing epel * Define cut down version of syslog-ng * Update inputs for `actions/download-artifact@v4` * Copy syslog-ng-amazon23.repo to artifacts dir * Temporary change ref to use new files * Update syslog-ng-amazon23.repo path * Add steps to ensure /etc/cron.d/ folder exist * Remove disabling rsyslog * Give cron.d 0755 access * Put in scripts for logrotate under cron.daily * Update external ethernet port name * Add `df -h` to monitor disk usage * Disable DOCKER_BUILDKIT for docker build * Make Makefile.nitro run quietly * Comment out uid2 build * Use .github/workflows/publish-aws-nitro-enclave-docker.yaml for build_aws_eif * Remove --quiet mode * Update start.sh * Install libxcrypt-compat * Modify scripts/aws/entrypoint.sh to match AL2023 metadata standard * Use echo $SECURITY_CREDS to get AWS keys * Convert .github/workflows/publish-aws-nitro-enclave-docker.yaml to main --- .github/actions/build_ami/action.yaml | 5 +- .github/actions/build_aws_eif/action.yaml | 2 + .../publish-aws-nitro-enclave-docker.yaml | 1 - scripts/aws/entrypoint.sh | 17 ++--- scripts/aws/logrotate/logrotateDaily | 8 +++ scripts/aws/sockd.conf | 2 +- scripts/aws/start.sh | 7 +- .../syslog-ng/server/syslog-ng-amazon23.repo | 10 +++ .../uid2-operator-ami/ansible/playbook.yml | 67 +++++++++++-------- scripts/aws/uid2-operator-ami/build.pkr.hcl | 2 +- scripts/aws/uid2-operator-ami/source.pkr.hcl | 3 +- 11 files changed, 75 insertions(+), 49 deletions(-) create mode 100644 scripts/aws/logrotate/logrotateDaily create mode 100644 scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo diff --git a/.github/actions/build_ami/action.yaml b/.github/actions/build_ami/action.yaml index 83e16522f..a614a943d 100644 --- a/.github/actions/build_ami/action.yaml +++ b/.github/actions/build_ami/action.yaml @@ -71,16 +71,13 @@ runs: - name: Get EIF for Run ${{ inputs.operator_run_number }} id: get_eif_for_run - uses: actions/download-artifact@v4 + uses: dawidd6/action-download-artifact@v6 if: ${{ inputs.operator_release == '' }} with: - github_token: ${{ inputs.github_token }} - repo: IABTechLab/uid2-operator name: 'aws-${{ inputs.identity_scope }}-deployment-files-.*' name_is_regexp: true run_id: ${{ inputs.operator_run_number }} skip_unpack: true - path: ./download/artifacts - name: Unzip artifacts if: ${{ inputs.operator_release == '' }} diff --git a/.github/actions/build_aws_eif/action.yaml b/.github/actions/build_aws_eif/action.yaml index 55663e497..c84ee2b68 100644 --- a/.github/actions/build_aws_eif/action.yaml +++ b/.github/actions/build_aws_eif/action.yaml @@ -68,8 +68,10 @@ runs: cp ./scripts/aws/syslog-ng/syslog-ng-server.conf ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/syslog-ng/server/syslog-ng-pubkey.gpg ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/syslog-ng/server/syslog-ng-4.6.0-1.el7.x86_64.rpm ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/logrotate/operator-logrotate.conf ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/logrotate/logrotate ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/logrotate/logrotateDaily ${ARTIFACTS_OUTPUT_DIR}/ docker cp amazonlinux:/sockd ${ARTIFACTS_OUTPUT_DIR}/ docker cp amazonlinux:/vsockpx ${ARTIFACTS_OUTPUT_DIR}/ diff --git a/.github/workflows/publish-aws-nitro-enclave-docker.yaml b/.github/workflows/publish-aws-nitro-enclave-docker.yaml index 374267e90..7d8611d0a 100644 --- a/.github/workflows/publish-aws-nitro-enclave-docker.yaml +++ b/.github/workflows/publish-aws-nitro-enclave-docker.yaml @@ -302,4 +302,3 @@ jobs: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/aws-euid-deployment-artifacts-${{ needs.start.outputs.new_version }}.zip ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/manifests/aws-uid2-enclave-id-${{ needs.start.outputs.new_version }}.txt ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/manifests/aws-euid-enclave-id-${{ needs.start.outputs.new_version }}.txt - diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 73f5debae..2b49660cc 100644 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -18,7 +18,8 @@ echo "Starting syslog-ng..." # -- load env vars via proxy echo "Loading env vars via proxy..." -USER_DATA=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/user-data) +TOKEN=$(curl -x socks5h://127.0.0.1:3305 --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600") +USER_DATA=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/user-data --header "X-aws-ec2-metadata-token: $TOKEN") if [ "${IDENTITY_SCOPE}" = "UID2" ]; then UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "uid2-operator-config-key") elif [ "${IDENTITY_SCOPE}" = "EUID" ]; then @@ -34,17 +35,17 @@ echo "UID2_CONFIG_SECRET_KEY=${UID2_CONFIG_SECRET_KEY}" echo "CORE_BASE_URL=${CORE_BASE_URL}" echo "OPTOUT_BASE_URL=${OPTOUT_BASE_URL}" -export AWS_REGION_NAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/dynamic/instance-identity/document/ | jq -r ".region") +export AWS_REGION_NAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r ".region") echo "AWS_REGION_NAME=${AWS_REGION_NAME}" echo "127.0.0.1 secretsmanager.${AWS_REGION_NAME}.amazonaws.com" >> /etc/hosts -IAM_ROLE=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/iam/security-credentials/) +IAM_ROLE=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/iam/security-credentials/ --header "X-aws-ec2-metadata-token: $TOKEN") echo "IAM_ROLE=${IAM_ROLE}" -CREDS_ENDPOINT="http://169.254.169.254/latest/meta-data/iam/security-credentials/${IAM_ROLE}" -export AWS_ACCESS_KEY_ID=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".AccessKeyId") -export AWS_SECRET_KEY=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".SecretAccessKey") -export AWS_SESSION_TOKEN=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".Token") +SECURITY_CREDS=$(curl -s -x socks5h://127.0.0.1:3305 "http://169.254.169.254/latest/meta-data/iam/security-credentials/${IAM_ROLE}" --header "X-aws-ec2-metadata-token: $TOKEN") +export AWS_ACCESS_KEY_ID=$(echo $SECURITY_CREDS | jq -r ".AccessKeyId") +export AWS_SECRET_KEY=$(echo $SECURITY_CREDS | jq -r ".SecretAccessKey") +export AWS_SESSION_TOKEN=$(echo $SECURITY_CREDS | jq -r ".Token") # -- load configs via proxy echo "Loading config overrides..." @@ -95,7 +96,7 @@ fi cat "${FINAL_CONFIG}" -HOSTNAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/local-hostname) +HOSTNAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/local-hostname --header "X-aws-ec2-metadata-token: $TOKEN") echo "HOSTNAME=${HOSTNAME}" # -- set pwd to /app so we can find default configs diff --git a/scripts/aws/logrotate/logrotateDaily b/scripts/aws/logrotate/logrotateDaily new file mode 100644 index 000000000..967932eec --- /dev/null +++ b/scripts/aws/logrotate/logrotateDaily @@ -0,0 +1,8 @@ +#!/bin/sh + +/usr/sbin/logrotate -s /var/lib/logrotate/logrotate.status /etc/logrotate.conf +EXITVALUE=$? +if [ $EXITVALUE != 0 ]; then + /usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE]" +fi +exit 0 diff --git a/scripts/aws/sockd.conf b/scripts/aws/sockd.conf index 69c6264f8..6e8814445 100644 --- a/scripts/aws/sockd.conf +++ b/scripts/aws/sockd.conf @@ -1,5 +1,5 @@ internal: 127.0.0.1 port = 3306 -external: eth0 +external: ens5 user.notprivileged: ec2-user clientmethod: none socksmethod: none diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh index 705acb80b..a9a08a7ef 100644 --- a/scripts/aws/start.sh +++ b/scripts/aws/start.sh @@ -4,11 +4,12 @@ echo "$HOSTNAME" > /etc/uid2operator/HOSTNAME EIF_PATH=${EIF_PATH:-/opt/uid2operator/uid2operator.eif} IDENTITY_SCOPE=${IDENTITY_SCOPE:-$(cat /opt/uid2operator/identity_scope.txt)} CID=${CID:-42} -AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ | jq -r '.region') +TOKEN=$(curl --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600") +AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r '.region') if [ "$IDENTITY_SCOPE" = 'UID2' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "uid2-operator-config-key") + UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data/ --header "X-aws-ec2-metadata-token: $TOKEN" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "uid2-operator-config-key") elif [ "$IDENTITY_SCOPE" = 'EUID' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "euid-operator-config-key") + UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data/ --header "X-aws-ec2-metadata-token: $TOKEN" | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "euid-operator-config-key") else echo "Unrecognized IDENTITY_SCOPE $IDENTITY_SCOPE" exit 1 diff --git a/scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo b/scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo new file mode 100644 index 000000000..7f93985a1 --- /dev/null +++ b/scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo @@ -0,0 +1,10 @@ +[copr:copr.fedorainfracloud.org:czanik:syslog-ng-amazon23] +name=Copr repo for syslog-ng-amazon23 owned by czanik +baseurl=https://download.copr.fedorainfracloud.org/results/czanik/syslog-ng-amazon23/amazonlinux-2023-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/czanik/syslog-ng-amazon23/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 diff --git a/scripts/aws/uid2-operator-ami/ansible/playbook.yml b/scripts/aws/uid2-operator-ami/ansible/playbook.yml index f80f5f013..36e3d4a8f 100644 --- a/scripts/aws/uid2-operator-ami/ansible/playbook.yml +++ b/scripts/aws/uid2-operator-ami/ansible/playbook.yml @@ -9,13 +9,13 @@ tasks: - name: Update all base packages - ansible.builtin.yum: + ansible.builtin.dnf: update_cache: yes name: '*' state: latest - name: Install Netcat - ansible.builtin.yum: + ansible.builtin.dnf: name: nmap-ncat state: latest @@ -29,6 +29,11 @@ path: /opt/uid2operator state: directory + - name: Install libxcrypt-compat + ansible.builtin.dnf: + name: libxcrypt-compat + state: latest + - name: vsock-proxy config ansible.builtin.copy: src: /tmp/artifacts/proxies.host.yaml @@ -115,10 +120,17 @@ dest: /etc/systemd/system/uid2operator.service remote_src: yes - - name: Ensure nitro-cli is installed to the latest version - ansible.builtin.command: amazon-linux-extras install aws-nitro-enclaves-cli + - name: Install AWS Nitro Enclaves CLI + ansible.builtin.dnf: + name: aws-nitro-enclaves-cli + state: latest + + - name: Install AWS Nitro Enclaves Devel CLI + ansible.builtin.dnf: + name: aws-nitro-enclaves-cli-devel + state: latest - - name: Ensure nitro enclave allocator is enabled at boot + - name: Enable the Nitro Enclave allocator service at boot ansible.builtin.systemd: name: nitro-enclaves-allocator.service enabled: yes @@ -133,27 +145,17 @@ # ansible.builtin.systemd: # name: nitro-enclaves-allocator.service # state: restarted - - name: Install EPEL - ansible.builtin.command: amazon-linux-extras install epel - - name: Install ivykis - ansible.builtin.yum: - name: ivykis-0.36.2-2.el7 - state: present - - - name: Install libnet - ansible.builtin.yum: - name: libnet-1.1.6-7.amzn2.0.2 - state: present - - - name: Register Public key - ansible.builtin.command: rpmkeys --import /opt/uid2operator/syslog-ng-pubkey.gpg - - - name: Install package - ansible.builtin.command: rpm -U /opt/uid2operator/syslog-ng-4.6.0-1.el7.x86_64.rpm + - name: Define cutdown version of syslog-ng + ansible.builtin.copy: + src: /tmp/artifacts/syslog-ng-amazon23.repo + dest: /etc/yum.repos.d/ + remote_src: yes - - name: unregister Public key - ansible.builtin.command: rpm -e gpg-pubkey-c57846f4-65a8cf14 + - name: Install syslog-ng + ansible.builtin.dnf: + name: syslog-ng + state: latest - name: syslog-ng config ansible.builtin.copy: @@ -166,6 +168,12 @@ src: /tmp/artifacts/operator-logrotate.conf dest: /etc/logrotate.d/operator-logrotate.conf remote_src: yes + + - name: Ensure /etc/cron.d/ folder exist + ansible.builtin.file: + path: /etc/cron.d + state: directory + mode: 0755 - name: Run logrotate under cron.d ansible.builtin.copy: @@ -174,11 +182,12 @@ mode: 0644 remote_src: yes - - name: Ensure rsyslog is disabled at boot - ansible.builtin.systemd: - name: rsyslog - state: stopped - enabled: false + - name: Run logrotate under cron.daily + ansible.builtin.copy: + src: /tmp/artifacts/logrotateDaily + dest: /etc/cron.daily/logrotate + mode: 0700 + remote_src: yes - name: Ensure syslog-ng is enabled at boot ansible.builtin.systemd: diff --git a/scripts/aws/uid2-operator-ami/build.pkr.hcl b/scripts/aws/uid2-operator-ami/build.pkr.hcl index 36d43390e..72addf7c6 100644 --- a/scripts/aws/uid2-operator-ami/build.pkr.hcl +++ b/scripts/aws/uid2-operator-ami/build.pkr.hcl @@ -8,7 +8,7 @@ build { provisioner "ansible" { playbook_file = "./ansible/playbook.yml" - extra_arguments = [ "--scp-extra-args", "'-O'", "--version", "9.6.1" ] + extra_arguments = [ "--scp-extra-args", "'-O'" ] } post-processor "manifest" { diff --git a/scripts/aws/uid2-operator-ami/source.pkr.hcl b/scripts/aws/uid2-operator-ami/source.pkr.hcl index b12766b0e..2c36b9086 100644 --- a/scripts/aws/uid2-operator-ami/source.pkr.hcl +++ b/scripts/aws/uid2-operator-ami/source.pkr.hcl @@ -3,9 +3,8 @@ source "amazon-ebs" "linux" { # source parameters source_ami_filter { filters = { - name = "amzn2-ami-hvm-*-x86_64-ebs" + name = "al2023-ami-2023*-x86_64" root-device-type = "ebs" - virtualization-type = "hvm" } most_recent = true owners = ["amazon"]