aws-refarch-wordpress-03-efsfilesystem.yaml

---
AWSTemplateFormatVersion: 2010-09-09

Description: Reference Architecture to host WordPress on AWS - Creates EFS file system

Metadata:

  Authors:
    Description: Darryl Osborne (darrylo@amazon.com)
  License:
    Description: 'Copyright 2018 Amazon.com, Inc. and its affiliates. All Rights Reserved.
      Licensed under the Amazon Software License (the "License"). You may not use this file
      except in compliance with the License. A copy of the License is located at
      http://aws.amazon.com/asl/
      or in the "license" file accompanying this file. This file is distributed on an "AS IS"
      BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
      License for the specific language governing permissions and limitations under the License.'

  AWS::CloudFormation::Interface:

    ParameterGroups:
    - Label:
        default: Amazon EFS Parameters
      Parameters:
        - Growth
        - InstanceType
        - EC2KeyName
        - PerformanceMode
        - EncrpytedBoolean
        - Cmk
        - SecurityGroup
        - NumberOfSubnets
        - Subnet
    ParameterLabels:
      EncrpytedBoolean:
        default: Encryption state
      Cmk:
        default: KMS Key
      Growth:
        default: Add data (GiB)
      InstanceType:
        default: Instance Type
      EC2KeyName:
        default: Existing Key Pair
      NumberOfSubnets:
        default: Number of subnets
      PerformanceMode:
        default: Performance Mode
      SecurityGroup:
        default: EFS Security Group
      Subnet:
        default: Subnets

Parameters:

  EncrpytedBoolean:
    AllowedValues:
      - True
      - False
    Default: True
    Description: Create an encrypted Amazon EFS file system.
    Type: String
  Cmk:
    Description: An existing AWS KMS Customer Master Key (CMK) to encrypt file system
    Type: String
  Growth:
    ConstraintDescription: Must be an integer.
    Default: 0
    Description: Amount of dummy data (GiB) to add to the file system (max 6144 GiB). Amazon EFS storage charges apply. 
    MaxValue: 6144
    MinValue: 0
    Type: Number
  InstanceType:
    AllowedValues:
      - t2.nano
      - t2.micro
      - t2.small
      - t2.medium
      - t2.large
      - t2.xlarge
      - t2.2xlarge
      - m3.medium
      - m3.large
      - m3.xlarge
      - m3.2xlarge
      - m4.large
      - m4.xlarge
      - m4.2xlarge
      - m4.4xlarge
      - m4.10xlarge
      - m4.16xlarge
      - c3.large
      - c3.xlarge
      - c3.2xlarge
      - c3.4xlarge
      - c3.8xlarge
      - c4.large
      - c4.xlarge
      - c4.2xlarge
      - c4.4xlarge
      - c4.8xlarge
      - c5.large
      - c5.xlarge
      - c5.2xlarge
      - c5.4xlarge
      - c5.8xlarge
      - r3.large
      - r3.xlarge
      - r3.2xlarge
      - r3.4xlarge
      - r3.8xlarge
      - r4.large
      - r4.xlarge
      - r4.2xlarge
      - r4.4xlarge
      - r4.8xlarge
      - r4.16xlarge
      - i3.large
      - i3.xlarge
      - i3.2xlarge
      - i3.4xlarge
      - i3.8xlarge
      - i3.16xlarge
      - d2.xlarge
      - d2.2xlarge
      - d2.4xlarge
      - d2.8xlarge
      - p2.xlarge
      - p2.8xlarage
      - p2.16xlarge
      - g3.4xlarge
      - g3.8xlarge
      - g3.16xlarge
      - f1.2xlarge
      - f1.16xlarge
      - x1.16xlarge
      - x1.32xlarge
    ConstraintDescription: Must be a valid Amazon EC2 instance type.
    Default: r4.large
    Description: The Amazon EC2 instance type that adds data to the file system.
    Type: String
  EC2KeyName:
    Description: Name of an existing EC2 key pair
    Type: AWS::EC2::KeyPair::KeyName
  NumberOfSubnets:
    AllowedValues:
    - 2
    - 3
    - 4
    - 5
    - 6
    Default: 3
    Description: Number of subnets. This must match your selections in the list of Subnets below.
    Type: String 
  PerformanceMode:
    AllowedValues:
      - generalPurpose
      - maxIO
    Default: generalPurpose
    Description: Select the performance mode of the file system.
    Type: String
  EfsCopyDataFrom:
    Description: Clone EFS from existing EFS
    Type: String
  WPDirectory:
    Description: Wordpress directory
    Type: String
  SecurityGroup:
    Description: Select the Amazon EFS security group.
    Type: AWS::EC2::SecurityGroup::Id   
  Subnet:
    Description: Select existing subnets.
    Type: List<AWS::EC2::Subnet::Id>

Conditions:
  
  NumberOfSubnets1:
      !Equals [ 1, !Ref NumberOfSubnets ]
  NumberOfSubnets2:
      !Equals [ 2, !Ref NumberOfSubnets ]
  NumberOfSubnets3:
      !Equals [ 3, !Ref NumberOfSubnets ]
  NumberOfSubnets4:
      !Equals [ 4, !Ref NumberOfSubnets ]
  NumberOfSubnets5:
      !Equals [ 5, !Ref NumberOfSubnets ]
  NumberOfSubnets6:
      !Equals [ 6, !Ref NumberOfSubnets ]
  Subnet0: !Or
    - !Condition NumberOfSubnets1
    - !Condition NumberOfSubnets2
    - !Condition NumberOfSubnets3
    - !Condition NumberOfSubnets4
    - !Condition NumberOfSubnets5
    - !Condition NumberOfSubnets6
  Subnet1: !Or
    - !Condition NumberOfSubnets2
    - !Condition NumberOfSubnets3
    - !Condition NumberOfSubnets4
    - !Condition NumberOfSubnets5
    - !Condition NumberOfSubnets6
  Subnet2: !Or
    - !Condition NumberOfSubnets3
    - !Condition NumberOfSubnets4
    - !Condition NumberOfSubnets5
    - !Condition NumberOfSubnets6
  Subnet3: !Or
    - !Condition NumberOfSubnets4
    - !Condition NumberOfSubnets5
    - !Condition NumberOfSubnets6
  Subnet4: !Or
    - !Condition NumberOfSubnets5
    - !Condition NumberOfSubnets6
  Subnet5: !Condition NumberOfSubnets6  
  UseAWS-ManagedCMK:
    !Equals ['', !Ref Cmk]

Mappings:

  RegionMap:
    ap-northeast-1:
      AMI: ami-2a69be4c
    ap-northeast-2:
      AMI: ami-9bec36f5
    ap-south-1:
      AMI: ami-4fc58420
    ap-southeast-1:
      AMI: ami-0797ea64
    ap-southeast-2:
      AMI: ami-8536d6e7
    ca-central-1:
      AMI: ami-fd55ec99
    eu-central-1:
      AMI: ami-c7ee5ca8
    eu-west-1:
      AMI: ami-acd005d5
    eu-west-2:
      AMI: ami-1a7f6d7e
    sa-east-1:
      AMI: ami-f1344b9d
    us-east-1:
      AMI: ami-8c1be5f6
    us-east-2:
      AMI: ami-c5062ba0
    us-west-1:
      AMI: ami-02eada62
    us-west-2:
      AMI: ami-e689729e

Resources:

  ElasticFileSystem:
    Type: AWS::EFS::FileSystem
    Properties:
      Encrypted: !Ref EncrpytedBoolean
      KmsKeyId:
        !If [ UseAWS-ManagedCMK, !Ref 'AWS::NoValue', !Ref Cmk ]
      FileSystemTags:
        - Key: Name
          Value: !Join [ '', [ 'WordPress / ', !Ref 'AWS::StackName' ] ]
      PerformanceMode: !Ref PerformanceMode
  ElasticFileSystemMountTarget0:
    Condition: Subnet0
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 0, !Ref Subnet ]
  ElasticFileSystemMountTarget1:
    Condition: Subnet1
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 1, !Ref Subnet ]
  ElasticFileSystemMountTarget2:
    Condition: Subnet2
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 2, !Ref Subnet ]
  ElasticFileSystemMountTarget3:
    Condition: Subnet3
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 3, !Ref Subnet ]
  ElasticFileSystemMountTarget4:
    Condition: Subnet4
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 4, !Ref Subnet ]
  ElasticFileSystemMountTarget5:
    Condition: Subnet5
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups:
      - !Ref SecurityGroup
      SubnetId: !Select [ 5, !Ref Subnet ]
  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref InstanceRole
  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: efs-create-file-system-with-storage
        PolicyDocument:
          Version: 2012-10-17
          Statement:
          - Effect: Allow
            Action:
              - autoscaling:DescribeAutoScalingGroups
              - autoscaling:DescribeAutoScalingInstances
              - autoscaling:DescribePolicies
              - autoscaling:UpdateAutoScalingGroup
            Resource: '*'
  AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      Cooldown: 60
      HealthCheckGracePeriod: 120
      HealthCheckType: EC2
      LaunchConfigurationName: !Ref LaunchConfiguration
      MaxSize: 1
      MinSize: 0
      DesiredCapacity: 1
      Tags:
        - Key: Name
          Value: !Join [ '', [ 'EFS ', !Ref ElasticFileSystem, ' data load... will auto terminate.' ] ]
          PropagateAtLaunch: true
      VPCZoneIdentifier:
        !If
          [ NumberOfSubnets1,
          [ !Select [ 0, !Ref Subnet ] ],
          !If
            [ NumberOfSubnets2,
            [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ] ],
            !If
              [ NumberOfSubnets3,
              [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ] ],
              !If
                [ NumberOfSubnets4,
                [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ] ],
                !If
                  [ NumberOfSubnets5,
                  [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ] ],
                  [ !Select [ 0, !Ref Subnet ], !Select [ 1, !Ref Subnet ], !Select [ 2, !Ref Subnet ], !Select [ 3, !Ref Subnet ], !Select [ 4, !Ref Subnet ], !Select [ 5, !Ref Subnet ] ]
                  ]
                ]
              ]
            ]
          ]
    CreationPolicy:
      ResourceSignal:
        Count: 0
        Timeout: PT12H
    UpdatePolicy:
      AutoScalingReplacingUpdate:
        WillReplace: true
  AutoScalingPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      AutoScalingGroupName: !Ref AutoScalingGroup
      Cooldown: 60
      PolicyType: SimpleScaling
      ScalingAdjustment: 1
  LaunchConfiguration:
    Type: AWS::AutoScaling::LaunchConfiguration
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          efs_add_storage:
            - efs-add-storage
        efs-add-storage:
          files:              
            /tmp/efs-add-storage.sh:
              content: !Sub |
                #!/bin/bash -x

                FILE_SYSTEM_ID=$1
                DATA_DIRECTORY=$2
                GROWTH=$3
                COPY_SYSTEM_ID=$4
                WP_DIR=$5

                if [ $# -lt 3 ]; then
                  echo "Invalid # of arguments. Require: file system id, data directory, file system growth (GiB) "
                  exit 0
                fi

                # get region from instance meta-data
                availabilityzone=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
                region=${!availabilityzone:0:-1}

                # get instance id
                instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)

                # get autoscaling group name
                asg_name=$(aws autoscaling describe-auto-scaling-instances --instance-ids $instance_id --region $region --output text --query 'AutoScalingInstances[0].AutoScalingGroupName')

                # set the number of threads to the number of vcpus
                threads=$(( $(nproc --all) * 8 ))

                # wait for file system DNS name to be propagated
                results=1
                while [[ $results != 0 ]]; do
                  nslookup $FILE_SYSTEM_ID.efs.$region.amazonaws.com
                  results=$?
                  if [[ results = 1 ]]; then
                    sleep 30
                  fi
                done

                # mount file system
                sudo mkdir -p /$FILE_SYSTEM_ID
                sudo chown ec2-user:ec2-user /$FILE_SYSTEM_ID
                sudo mountpoint -q /$FILE_SYSTEM_ID || sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $FILE_SYSTEM_ID.efs.$region.amazonaws.com:/ /$FILE_SYSTEM_ID

                # create directory if not exists
                sudo mkdir -p /$FILE_SYSTEM_ID/$DATA_DIRECTORY
                sudo chown ec2-user:ec2-user /$FILE_SYSTEM_ID/$DATA_DIRECTORY

                # dd 1GiB files to file system to match DATA_SIZE
                files=$GROWTH
                if [ $(( $files / $threads )) == 0 ];
                  then
                    runs=0
                    parallel_threads=$(( $files % $threads ))
                  else
                    runs=$(( $files / $threads ))
                    parallel_threads=$threads
                fi
                while [ $runs -ge 0 ]; do
                  if [ $runs == 0 ];
                    then
                      parallel_threads=$(( $files % $threads ))
                      seq 0 $(( $parallel_threads - 1 )) | parallel --will-cite -j $parallel_threads --compress dd if=/dev/zero of=/$FILE_SYSTEM_ID/$DATA_DIRECTORY/1G-dd-$(date +%Y%m%d%H%M%S.%3N)-{} bs=1M count=1024 oflag=sync
                      runs=$(($runs-1))
                    else
                      seq 0 $(( $parallel_threads - 1 )) | parallel --will-cite -j $parallel_threads --compress dd if=/dev/zero of=/$FILE_SYSTEM_ID/$DATA_DIRECTORY/1G-dd-$(date +%Y%m%d%H%M%S.%3N)-{} bs=1M count=1024 oflag=sync
                      runs=$(($runs-1))
                  fi
                done

                #Sync data from 
                
                if [[ ! -z $COPY_SYSTEM_ID ]];
                  then
                    sudo mkdir -p /$COPY_SYSTEM_ID
                    sudo chown ec2-user:ec2-user /$COPY_SYSTEM_ID
                    sudo mountpoint -q /$COPY_SYSTEM_ID || sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $COPY_SYSTEM_ID.efs.$region.amazonaws.com:/ /$COPY_SYSTEM_ID
                    COPY_SOURCE=/$COPY_SYSTEM_ID
                    if [ -d "/$COPY_SOURCE/$WP_DIR" ]; then
                      COPY_SOURCE=/$COPY_SYSTEM_ID/$WP_DIR #If the wordpress folder is there, ensure we get only the contents
                    fi
                    sudo mkdir -p /$FILE_SYSTEM_ID/$WP_DIR
                    sudo rsync -r $COPY_SOURCE/* /$FILE_SYSTEM_ID/$WP_DIR/
                fi

                # set ASG to zero which terminates instance
                aws autoscaling update-auto-scaling-group --auto-scaling-group-name $asg_name --desired-capacity 0 --region $region

              mode: 000777
              owner: root
              group: root
    Properties:
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            DeleteOnTermination: true
            VolumeSize: 10
            VolumeType: gp2
      IamInstanceProfile: !Ref InstanceProfile
      ImageId: !FindInMap [ RegionMap, !Ref 'AWS::Region', AMI ]
      InstanceMonitoring: true
      InstanceType: !Ref InstanceType
      KeyName: !Ref EC2KeyName
      SecurityGroups:
      - !Ref SecurityGroup
      UserData:
        "Fn::Base64":
          !Join [
            "",[
            "#cloud-config\n",
            "repo_update: true\n",
            "repo_upgrade: all\n",
            "\n",
            "packages:\n",
            "- parallel\n",
            "\n",
            "runcmd:\n",
            "- yum --enablerepo=epel install nload -y\n",
            "- ntpstat\n",
            "- /opt/aws/bin/cfn-init --configsets efs_add_storage --verbose --stack ", !Ref 'AWS::StackName', " --resource LaunchConfiguration --region ", !Ref 'AWS::Region',"\n",
            "- /tmp/efs-add-storage.sh ", !Ref ElasticFileSystem, " throughput_data ", !Ref Growth, " ", !Ref EfsCopyDataFrom, " ", !Ref WPDirectory, "\n",
            "- /opt/aws/bin/cfn-signal -e $? --stack ", !Ref 'AWS::StackName', " --resource AutoScalingGroup --region ", !Ref 'AWS::Region',"\n"
            ]
          ]
  EfsSizeMonitorFunction:
    Type: AWS::Lambda::Function
    Properties: 
      Code:
        ZipFile: !Sub |
          import boto3
          import os
          import sys

          def handler(event, context):
              if not os.environ.get('filesystemid'):
                  print "Unable to get the environment variable filesystemid"
                  sys.exit(1)
              else:
                  filesystemid = os.environ.get('filesystemid')

              if not os.environ.get('region'):
                  print "Unable to get the environment variable region"
                  sys.exit(1)
              else:
                  region = os.environ.get('region')

              def efs_get_size():
                  client = boto3.client('efs')
                  response = client.describe_file_systems(FileSystemId=filesystemid)
                  k = response['FileSystems'][0]['SizeInBytes']['Value']
                  return k

              def cloudwatch_put_metric():
                  client = boto3.client('cloudwatch')
                  client.put_metric_data(
                      MetricData=[
                          {
                              'MetricName': 'SizeInBytes',
                              'Dimensions': [
                                  {
                                      'Name': 'FileSystemId',
                                      'Value': filesystemid
                                  },
                              ],
                              'Unit': 'None',
                              'Value': efs_get_size()
                          },
                      ],
                      Namespace='Custom/EFS'
                  )
                  print('CloudWatch metric SizeInBytes sucessfully updated.')

              cloudwatch_put_metric()
      Description: Lambda function to update the SizeInBytes EFS CloudWatch metric
      Environment:
        Variables:
          filesystemid: !Ref ElasticFileSystem
          region: !Ref 'AWS::Region'
      FunctionName: !Join [ '', [ 'efs-', !Ref ElasticFileSystem, '-size-monitor' ] ]
      Handler: index.handler
      Role: !GetAtt LambdaRole.Arn
      Runtime: python2.7
      Timeout: 60
  LambdaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/CloudWatchFullAccess
      - arn:aws:iam::aws:policy/AmazonElasticFileSystemReadOnlyAccess
  EfsLambdaPermission: 
    Type: AWS::Lambda::Permission
    Properties: 
      FunctionName: !Ref EfsSizeMonitorFunction
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt EfsSizeMonitorEvent.Arn
  EfsSizeMonitorEvent:
    Type: AWS::Events::Rule
    Properties: 
      Description: Scheduled event to update SizeInBytes EFS CloudWatch metric
      Name: !Join [ '', [ 'efs-', !Ref ElasticFileSystem, '-size-monitor-scheduled-event' ] ]
      ScheduleExpression: rate(1 minute)
      State: ENABLED
      Targets:
        - Arn: !GetAtt EfsSizeMonitorFunction.Arn
          Id: 1

Outputs:
  ElasticFileSystem:
    Value: !Ref ElasticFileSystem
  ElasticFileSystemDnsName:
    Description: DNS name for the Amazon EFS file system.
    Value: !Join [ '.', [ !Ref ElasticFileSystem, 'efs', !Ref 'AWS::Region', 'amazonaws', 'com' ] ]
  ElasticFileSystemMountCommand:
    Description: Mount command for mounting the Amazon EFS file system.
    Value: !Join [ '', [ 'sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 ', !Join [ '.', [ !Ref ElasticFileSystem, 'efs', !Ref 'AWS::Region', 'amazonaws', 'com:/', '/', !Ref ElasticFileSystem ] ] ] ]