UID2-3331 Update ami from al2 to al2023 (#641)

* Update AL source

* Remove unnecessary arguments

* Update installing aws-nitro-enclaves-cli

* Use dnf instead of yum

* Remove the step for installing epel

* Define cut down version of syslog-ng

* Update inputs for `actions/download-artifact@v4`

* Copy syslog-ng-amazon23.repo to artifacts dir

* Temporary change ref to use new files

* Update syslog-ng-amazon23.repo path

* Add steps to ensure /etc/cron.d/ folder exist

* Remove disabling rsyslog

* Give cron.d 0755 access

* Put in scripts for logrotate under cron.daily

* Update external ethernet port name

* Add `df -h` to monitor disk usage

* Disable DOCKER_BUILDKIT for docker build

* Make Makefile.nitro run quietly

* Comment out uid2 build

* Use .github/workflows/publish-aws-nitro-enclave-docker.yaml for build_aws_eif

* Remove --quiet mode

* Update start.sh

* Install libxcrypt-compat

* Modify scripts/aws/entrypoint.sh to match AL2023 metadata standard

* Use echo $SECURITY_CREDS to get AWS keys

* Convert .github/workflows/publish-aws-nitro-enclave-docker.yaml to main

.github/actions/build_ami/action.yaml

@@ -71,16 +71,13 @@ runs:
 
     - name: Get EIF for Run ${{ inputs.operator_run_number }}
       id: get_eif_for_run
-      uses: actions/download-artifact@v4
+      uses: dawidd6/action-download-artifact@v6
       if: ${{ inputs.operator_release == '' }}
       with:
-        github_token: ${{ inputs.github_token }}
-        repo: IABTechLab/uid2-operator
         name: 'aws-${{ inputs.identity_scope }}-deployment-files-.*'
         name_is_regexp: true
         run_id: ${{ inputs.operator_run_number }}
         skip_unpack: true
-        path: ./download/artifacts
 
     - name: Unzip artifacts
       if: ${{ inputs.operator_release == '' }}

.github/actions/build_aws_eif/action.yaml

@@ -68,8 +68,10 @@ runs:
         cp ./scripts/aws/syslog-ng/syslog-ng-server.conf                                                          ${ARTIFACTS_OUTPUT_DIR}/
         cp ./scripts/aws/syslog-ng/server/syslog-ng-pubkey.gpg                                                    ${ARTIFACTS_OUTPUT_DIR}/
         cp ./scripts/aws/syslog-ng/server/syslog-ng-4.6.0-1.el7.x86_64.rpm                                        ${ARTIFACTS_OUTPUT_DIR}/
+        cp ./scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo                                                 ${ARTIFACTS_OUTPUT_DIR}/
         cp ./scripts/aws/logrotate/operator-logrotate.conf                                                        ${ARTIFACTS_OUTPUT_DIR}/
         cp ./scripts/aws/logrotate/logrotate                                                                      ${ARTIFACTS_OUTPUT_DIR}/
+        cp ./scripts/aws/logrotate/logrotateDaily                                                                 ${ARTIFACTS_OUTPUT_DIR}/
 
         docker cp amazonlinux:/sockd                                    ${ARTIFACTS_OUTPUT_DIR}/
         docker cp amazonlinux:/vsockpx                                  ${ARTIFACTS_OUTPUT_DIR}/

.github/workflows/publish-aws-nitro-enclave-docker.yaml

@@ -302,4 +302,3 @@ jobs:
             ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/aws-euid-deployment-artifacts-${{ needs.start.outputs.new_version }}.zip
             ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/manifests/aws-uid2-enclave-id-${{ needs.start.outputs.new_version }}.txt
             ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/manifests/aws-euid-enclave-id-${{ needs.start.outputs.new_version }}.txt
-

scripts/aws/entrypoint.sh

@@ -18,7 +18,8 @@ echo "Starting syslog-ng..."
 # -- load env vars via proxy
 echo "Loading env vars via proxy..."
 
-USER_DATA=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/user-data)
+TOKEN=$(curl -x socks5h://127.0.0.1:3305 --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600")
+USER_DATA=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/user-data --header "X-aws-ec2-metadata-token: $TOKEN")
 if [ "${IDENTITY_SCOPE}" = "UID2" ]; then
   UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "uid2-operator-config-key")
 elif [ "${IDENTITY_SCOPE}" = "EUID" ]; then
@@ -34,17 +35,17 @@ echo "UID2_CONFIG_SECRET_KEY=${UID2_CONFIG_SECRET_KEY}"
 echo "CORE_BASE_URL=${CORE_BASE_URL}"
 echo "OPTOUT_BASE_URL=${OPTOUT_BASE_URL}"
 
-export AWS_REGION_NAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/dynamic/instance-identity/document/ | jq -r ".region")
+export AWS_REGION_NAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r ".region")
 echo "AWS_REGION_NAME=${AWS_REGION_NAME}"
 echo "127.0.0.1 secretsmanager.${AWS_REGION_NAME}.amazonaws.com" >> /etc/hosts
 
-IAM_ROLE=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/iam/security-credentials/)
+IAM_ROLE=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/iam/security-credentials/ --header "X-aws-ec2-metadata-token: $TOKEN")
 echo "IAM_ROLE=${IAM_ROLE}"
 
-CREDS_ENDPOINT="http://169.254.169.254/latest/meta-data/iam/security-credentials/${IAM_ROLE}"
-export AWS_ACCESS_KEY_ID=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".AccessKeyId")
-export AWS_SECRET_KEY=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".SecretAccessKey")
-export AWS_SESSION_TOKEN=$(curl -s -x socks5h://127.0.0.1:3305 "${CREDS_ENDPOINT}" | jq -r ".Token")
+SECURITY_CREDS=$(curl -s -x socks5h://127.0.0.1:3305 "http://169.254.169.254/latest/meta-data/iam/security-credentials/${IAM_ROLE}" --header "X-aws-ec2-metadata-token: $TOKEN")
+export AWS_ACCESS_KEY_ID=$(echo $SECURITY_CREDS | jq -r ".AccessKeyId")
+export AWS_SECRET_KEY=$(echo $SECURITY_CREDS | jq -r ".SecretAccessKey")
+export AWS_SESSION_TOKEN=$(echo $SECURITY_CREDS | jq -r ".Token")
 
 # -- load configs via proxy
 echo "Loading config overrides..."
@@ -95,7 +96,7 @@ fi
 
 cat "${FINAL_CONFIG}"
 
-HOSTNAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/local-hostname)
+HOSTNAME=$(curl -s -x socks5h://127.0.0.1:3305 http://169.254.169.254/latest/meta-data/local-hostname --header "X-aws-ec2-metadata-token: $TOKEN")
 echo "HOSTNAME=${HOSTNAME}"
 
 # -- set pwd to /app so we can find default configs

scripts/aws/logrotate/logrotateDaily

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+/usr/sbin/logrotate -s /var/lib/logrotate/logrotate.status /etc/logrotate.conf
+EXITVALUE=$?
+if [ $EXITVALUE != 0 ]; then
+    /usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE]"
+fi
+exit 0

scripts/aws/sockd.conf

@@ -1,5 +1,5 @@
 internal: 127.0.0.1 port = 3306
-external: eth0
+external: ens5
 user.notprivileged: ec2-user
 clientmethod: none
 socksmethod: none

scripts/aws/start.sh

@@ -4,11 +4,12 @@ echo "$HOSTNAME" > /etc/uid2operator/HOSTNAME
 EIF_PATH=${EIF_PATH:-/opt/uid2operator/uid2operator.eif}
 IDENTITY_SCOPE=${IDENTITY_SCOPE:-$(cat /opt/uid2operator/identity_scope.txt)}
 CID=${CID:-42}
-AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ | jq -r '.region')
+TOKEN=$(curl --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600")
+AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r '.region')
 if [ "$IDENTITY_SCOPE" = 'UID2' ]; then
-  UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "uid2-operator-config-key")
+  UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data/ --header "X-aws-ec2-metadata-token: $TOKEN" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "uid2-operator-config-key")
 elif [ "$IDENTITY_SCOPE" = 'EUID' ]; then
-  UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "euid-operator-config-key")
+  UID2_CONFIG_SECRET_KEY=$([[ "$(curl -s http://169.254.169.254/latest/user-data/ --header "X-aws-ec2-metadata-token: $TOKEN" | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\" ]] && echo ${BASH_REMATCH[1]} || echo "euid-operator-config-key")
 else
   echo "Unrecognized IDENTITY_SCOPE $IDENTITY_SCOPE"
   exit 1

scripts/aws/syslog-ng/server/syslog-ng-amazon23.repo

@@ -0,0 +1,10 @@
+[copr:copr.fedorainfracloud.org:czanik:syslog-ng-amazon23]
+name=Copr repo for syslog-ng-amazon23 owned by czanik
+baseurl=https://download.copr.fedorainfracloud.org/results/czanik/syslog-ng-amazon23/amazonlinux-2023-$basearch/
+type=rpm-md
+skip_if_unavailable=True
+gpgcheck=1
+gpgkey=https://download.copr.fedorainfracloud.org/results/czanik/syslog-ng-amazon23/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+enabled_metadata=1

scripts/aws/uid2-operator-ami/ansible/playbook.yml

@@ -9,13 +9,13 @@
 
   tasks:
   - name: Update all base packages
-    ansible.builtin.yum:
+    ansible.builtin.dnf:
       update_cache: yes
       name: '*'
       state: latest
 
   - name: Install Netcat
-    ansible.builtin.yum:
+    ansible.builtin.dnf:
       name: nmap-ncat
       state: latest
 
@@ -29,6 +29,11 @@
       path: /opt/uid2operator
       state: directory
 
+  - name: Install libxcrypt-compat
+    ansible.builtin.dnf:
+      name: libxcrypt-compat
+      state: latest
+
   - name: vsock-proxy config
     ansible.builtin.copy:
       src: /tmp/artifacts/proxies.host.yaml
@@ -115,10 +120,17 @@
       dest: /etc/systemd/system/uid2operator.service
       remote_src: yes
 
-  - name: Ensure nitro-cli is installed to the latest version
-    ansible.builtin.command: amazon-linux-extras install aws-nitro-enclaves-cli
+  - name: Install AWS Nitro Enclaves CLI
+    ansible.builtin.dnf:
+      name: aws-nitro-enclaves-cli
+      state: latest
+
+  - name: Install AWS Nitro Enclaves Devel CLI
+    ansible.builtin.dnf:
+      name: aws-nitro-enclaves-cli-devel
+      state: latest
 
-  - name: Ensure nitro enclave allocator is enabled at boot
+  - name: Enable the Nitro Enclave allocator service at boot
     ansible.builtin.systemd:
       name: nitro-enclaves-allocator.service
       enabled: yes
@@ -133,27 +145,17 @@
   #   ansible.builtin.systemd:
   #     name: nitro-enclaves-allocator.service
   #     state: restarted
-  - name: Install EPEL
-    ansible.builtin.command: amazon-linux-extras install epel
 
-  - name: Install ivykis
-    ansible.builtin.yum:
-      name: ivykis-0.36.2-2.el7
-      state: present
-
-  - name: Install libnet
-    ansible.builtin.yum:
-      name: libnet-1.1.6-7.amzn2.0.2
-      state: present
-
-  - name: Register Public key
-    ansible.builtin.command: rpmkeys --import /opt/uid2operator/syslog-ng-pubkey.gpg
-
-  - name: Install package
-    ansible.builtin.command: rpm -U /opt/uid2operator/syslog-ng-4.6.0-1.el7.x86_64.rpm
+  - name: Define cutdown version of syslog-ng
+    ansible.builtin.copy:
+      src: /tmp/artifacts/syslog-ng-amazon23.repo
+      dest: /etc/yum.repos.d/
+      remote_src: yes
 
-  - name: unregister Public key
-    ansible.builtin.command: rpm -e gpg-pubkey-c57846f4-65a8cf14
+  - name: Install syslog-ng
+    ansible.builtin.dnf:
+      name: syslog-ng
+      state: latest
 
   - name: syslog-ng config
     ansible.builtin.copy:
@@ -166,6 +168,12 @@
       src: /tmp/artifacts/operator-logrotate.conf
       dest: /etc/logrotate.d/operator-logrotate.conf
       remote_src: yes
+
+  - name: Ensure /etc/cron.d/ folder exist
+    ansible.builtin.file:
+      path: /etc/cron.d
+      state: directory
+      mode: 0755
 
   - name: Run logrotate under cron.d
     ansible.builtin.copy:
@@ -174,11 +182,12 @@
       mode: 0644
       remote_src: yes
 
-  - name: Ensure rsyslog is disabled at boot
-    ansible.builtin.systemd:
-      name: rsyslog
-      state: stopped
-      enabled: false
+  - name: Run logrotate under cron.daily
+    ansible.builtin.copy:
+      src: /tmp/artifacts/logrotateDaily
+      dest: /etc/cron.daily/logrotate
+      mode: 0700
+      remote_src: yes
 
   - name: Ensure syslog-ng is enabled at boot
     ansible.builtin.systemd:

scripts/aws/uid2-operator-ami/build.pkr.hcl

@@ -8,7 +8,7 @@ build {
 
   provisioner "ansible" {
     playbook_file = "./ansible/playbook.yml"
-    extra_arguments = [ "--scp-extra-args", "'-O'", "--version", "9.6.1" ]
+    extra_arguments = [ "--scp-extra-args", "'-O'" ]
   }
 
   post-processor "manifest" {

scripts/aws/uid2-operator-ami/source.pkr.hcl

@@ -3,9 +3,8 @@ source "amazon-ebs" "linux" {
   # source parameters
   source_ami_filter {
     filters = {
-      name                = "amzn2-ami-hvm-*-x86_64-ebs"
+      name                = "al2023-ami-2023*-x86_64"
       root-device-type    = "ebs"
-      virtualization-type = "hvm"
     }
     most_recent = true
     owners      = ["amazon"]