From 19a5cac67360e9838cdcf16baae3b07ab336952f Mon Sep 17 00:00:00 2001 From: fred-vogt-dod Date: Mon, 10 Oct 2022 16:32:43 -0700 Subject: [PATCH] Use separate EBS Volumes for /var, /home folders --- amazon-eks-al2.pkr.hcl | 80 +++++++++++++++++++++++++------------- scripts/partition-disks.sh | 70 +++++++++++++++++++-------------- variables.pkr.hcl | 44 ++++++++++++++++----- 3 files changed, 128 insertions(+), 66 deletions(-) diff --git a/amazon-eks-al2.pkr.hcl b/amazon-eks-al2.pkr.hcl index 3c8a83b..fc6dd8e 100644 --- a/amazon-eks-al2.pkr.hcl +++ b/amazon-eks-al2.pkr.hcl @@ -1,9 +1,3 @@ -locals { - timestamp = regex_replace(timestamp(), "[- TZ:]", "") - - target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}" -} - data "amazon-ami" "this" { filters = { architecture = var.source_ami_arch @@ -21,36 +15,68 @@ data "amazon-ami" "this" { region = var.aws_region } -source "amazon-ebs" "this" { - ami_block_device_mappings { - delete_on_termination = true - device_name = "/dev/sdb" - volume_size = var.data_volume_size - volume_type = "gp2" - encrypted = true +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") + + target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}" + + block_device_mappings = { + "/" = { + device_name = "/dev/xvda" + volume_size = var.root_volume_size + } + "/home" = { + device_name = "/dev/sdf" + volume_size = var.home_volume_size + } + "/var" = { + device_name = "/dev/sdg" + volume_size = var.var_volume_size + } + "/var/log" = { + device_name = "/dev/sdh" + volume_size = var.varlog_volume_size + } + "/var/log/audit" = { + device_name = "/dev/sdi" + volume_size = var.varlogaudit_volume_size + } + "/var/lib/docker" = { + device_name = "/dev/sdj" + volume_size = var.varlibdocker_volume_size + } } +} +source "amazon-ebs" "this" { ami_description = "EKS Kubernetes Worker AMI with AmazonLinux2 image" ami_name = local.target_ami_name ami_virtualization_type = "hvm" instance_type = var.instance_type - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/xvda" - volume_size = var.root_volume_size - volume_type = "gp2" - encrypted = true - kms_key_id = var.kms_key_id + dynamic "ami_block_device_mappings" { + for_each = local.block_device_mappings + + content { + device_name = ami_block_device_mappings.value.device_name + volume_size = ami_block_device_mappings.value.volume_size + delete_on_termination = true + volume_type = "gp3" + encrypted = true + } } - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/sdb" - volume_size = var.data_volume_size - volume_type = "gp2" - encrypted = true - kms_key_id = var.kms_key_id + dynamic "launch_block_device_mappings" { + for_each = local.block_device_mappings + + content { + device_name = launch_block_device_mappings.value.device_name + volume_size = launch_block_device_mappings.value.volume_size + delete_on_termination = true + volume_type = "gp3" + encrypted = true + kms_key_id = var.kms_key_id + } } encrypt_boot = var.encrypt_boot diff --git a/scripts/partition-disks.sh b/scripts/partition-disks.sh index ebc3d95..72a0a5f 100755 --- a/scripts/partition-disks.sh +++ b/scripts/partition-disks.sh @@ -17,21 +17,32 @@ set -o errexit # None ################################################################ migrate_and_mount_disk() { - local disk_name=$1 + local device_name=$1 local folder_path=$2 local mount_options=$3 local temp_path="/mnt${folder_path}" local old_path="${folder_path}-old" - # install an ext4 filesystem to the disk - mkfs -t ext4 ${disk_name} + # AWS EC2 API Block Device Mapping name to Linux NVME device name + disk_name="/dev/$(readlink "$device_name")" + + # partition the disk (single data partition) + parted -a optimal -s $disk_name \ + mklabel gpt \ + mkpart data xfs 0% 90% + + # wait for the disk to settle + sleep 5 + + # install an xfs filesystem to the disk + mkfs -t xfs "${disk_name}p1" # check if the folder already exists if [ -d "${folder_path}" ]; then FILE=$(ls -A ${folder_path}) >&2 echo $FILE mkdir -p ${temp_path} - mount ${disk_name} ${temp_path} + mount "${disk_name}p1" ${temp_path} # Empty folder give error on /* if [ ! -z "$FILE" ]; then cp -Rax ${folder_path}/* ${temp_path} @@ -42,7 +53,7 @@ migrate_and_mount_disk() { mkdir -p ${folder_path} # add the mount point to fstab and mount the disk - echo "UUID=$(blkid -s UUID -o value ${disk_name}) ${folder_path} ext4 ${mount_options} 0 1" >> /etc/fstab + echo "UUID=$(blkid -s UUID -o value "${disk_name}p1") ${folder_path} xfs ${mount_options} 0 1" >> /etc/fstab mount -a # if selinux is enabled restore the objects on it @@ -51,27 +62,28 @@ migrate_and_mount_disk() { fi } -disk_name='/dev/nvme1n1' - -# partition the disk -parted -a optimal -s $disk_name \ - mklabel gpt \ - mkpart var ext4 0% 20% \ - mkpart varlog ext4 20% 40% \ - mkpart varlogaudit ext4 40% 60% \ - mkpart home ext4 60% 70% \ - mkpart varlibdocker ext4 70% 90% - -# wait for the disks to settle -sleep 5 - -# migrate and mount the existing -migrate_and_mount_disk "${disk_name}p1" /var defaults,nofail,nodev -migrate_and_mount_disk "${disk_name}p2" /var/log defaults,nofail,nodev,nosuid -migrate_and_mount_disk "${disk_name}p3" /var/log/audit defaults,nofail,nodev,nosuid -migrate_and_mount_disk "${disk_name}p4" /home defaults,nofail,nodev,nosuid - -# Create folder instead of starting/stopping docker daemon -mkdir -p /var/lib/docker -chown -R root:docker /var/lib/docker -migrate_and_mount_disk "${disk_name}p5" /var/lib/docker defaults,nofail +# migrate and mount the existing folders to dedicated EBS Volumes +migrate_and_mount_disk "/dev/sdf" "/home" defaults,nofail,nodev,nosuid +migrate_and_mount_disk "/dev/sdg" "/var" defaults,nofail,nodev +migrate_and_mount_disk "/dev/sdh" "/var/log" defaults,nofail,nodev,nosuid +migrate_and_mount_disk "/dev/sdi" "/var/log/audit" defaults,nofail,nodev,nosuid +migrate_and_mount_disk "/dev/sdj" "/var/lib/docker" defaults,nofail + +# Resize on instance launch +cloud_init_script="/var/lib/cloud/scripts/per-boot/resize-disks.sh" +cat > "$cloud_init_script" <