Merge pull request #157 from RLOpenCatalyst/topic-release

1.19.0 build updates

cft-templates/Rstudio.yml

@@ -32,10 +32,6 @@ Parameters:
     Type: String
     Description: User Name for RStudio. Do not use 'root' and 'ec2-user'
     Default: rstudio
-  InitialPassword:
-    Type: String
-    Description: Password for RStudio. Please keep in your records as this will not be echoed in the CloudFormation Console
-    NoEcho: True
   KeyPair:
     Type: "AWS::EC2::KeyPair::KeyName"
     Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. If no key pairs exist, please create one from the button next to the dropdown. Please contact your Administrator if you are unable to create one.
@@ -64,6 +60,15 @@ Resources:
           - !Ref 'AWS::NoValue'
           - PolicyName: !Join ['-', [Ref: Namespace, 's3-studydata-policy']]
             PolicyDocument: !Ref IamPolicyDocument
+        - PolicyName: param-store-access
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: 'Allow'
+                Action:
+                  - 'ssm:GetParameter'
+                  - 'ssm:PutParameter'
+                Resource: !Sub 'arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/*'            
 
   InstanceProfile:
     Type: 'AWS::IAM::InstanceProfile'
@@ -92,6 +97,44 @@ Resources:
     CreationPolicy:
       ResourceSignal:
         Timeout: PT10M
+    Metadata:
+      AWS::CloudFormation::Init:
+        configSets:
+          default:
+            - config1
+            - config2
+        config1:
+          files:         
+            '/root/set_user_token.sh':
+              mode: '000755'
+              owner: 'ec2-user'
+              group: 'ec2-user'
+              content: !Sub |
+                #!/usr/bin/env bash
+                instance_id=$(curl -s "http://169.254.169.254/latest/meta-data/instance-id")
+                secret=`uuidgen`
+                echo "setting ${InitialUser} password and starting rstudio"
+                password=$(echo -n "$instance_id$secret" | sha256sum | awk '{print $1}')
+                echo "${InitialUser}:$password" | /usr/sbin/chpasswd
+                echo "Set rstudio-user password"
+                                             
+                sleep 10
+                public_key=$(curl http://localhost:8787/auth-public-key)
+                instance_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region")
+                aws ssm put-parameter --name "/RL/RG/rstudio/public-key/$instance_id" --value '{"secret":"'$secret'","public_key":"'$public_key'"}' --region $instance_region --type SecureString --overwrite 
+                echo "Stored rstudio public key in SSM"
+            '/var/log/rstudio.log':
+              content: "\n"
+              mode: '000664'
+              owner: 'root'
+              group: 'root'
+
+        config2:
+          commands:
+            02_set_user_token:
+              cwd: '/root'
+              command: '/root/set_user_token.sh'
+
     Properties:
       ImageId : '{{resolve:ssm:/RL/RG/StandardCatalog/RStudio}}'
       InstanceType: !Ref 'InstanceType'
@@ -115,15 +158,22 @@ Resources:
 
           #add user(s)
           sudo useradd -m -s /bin/bash ${InitialUser}
-          sudo echo ${InitialUser}:${InitialPassword} | chpasswd
           #Add user to the sudo group
-          sudo usermod -a -G wheel ${InitialUser}
+          sudo usermod -a -G wheel ${InitialUser}             
 
           #Download and execute bootstrap script
           aws s3 cp "${EnvironmentInstanceFiles}/get_bootstrap.sh" "/tmp"
           chmod 500 "/tmp/get_bootstrap.sh"
           /tmp/get_bootstrap.sh "${EnvironmentInstanceFiles}" '${S3Mounts}' "${InitialUser}"
 
+          # Add command to start on reboot
+          crontab -l 2>/dev/null > "/tmp/crontab"
+          echo '@reboot /root/set_user_token.sh 2>&1 >> /var/log/set_user_token.log' >> "/tmp/crontab"
+          crontab "/tmp/crontab"
+
+          # Run init script to create files
+          /opt/aws/bin/cfn-init --verbose --stack ${AWS::StackName} --resource RstudioEC2Instance --region ${AWS::Region}
+
           /opt/aws/bin/cfn-signal --exit-code 0 --resource RstudioEC2Instance --region ${AWS::Region} --stack ${AWS::StackName}
 
 Outputs:

cft-templates/ec2-jupyterLab.yml

@@ -0,0 +1,264 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Description: Sets up Jupyter Lab environment on AWS
+
+Parameters:
+  Namespace:
+    Type: String
+    Description: An environment name that will be prefixed to resource names
+  S3Mounts:
+    Type: String
+    Description: A JSON array of objects with name, bucket, and prefix properties used to mount data
+  IamPolicyDocument:
+    Type: String
+    Description: The IAM policy to be associated with the launched workstation
+  EnvironmentInstanceFiles:
+    Type: String
+    Description: >-
+      An S3 URI (starting with "s3://") that specifies the location of files to be copied to
+      the environment instance, including any bootstrap scripts
+  InstanceType:
+    Type: String
+    Description: Instance type for JupyterLab. Default is t3.medium.
+    AllowedValues:
+      - t2.medium
+      - t2.large
+      - t3.medium
+      - t3.large      
+    ConstraintDescription: Valid instance type in the t3 families
+    Default: t3.medium
+  ContainerName:
+    Type: String
+    Description: Docker image of the application to be launched
+    Default: relevancelab/jupiterlab_3.5.0:1.0.3
+  KeyPair:
+    Type: "AWS::EC2::KeyPair::KeyName"
+    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. If no key pairs exist, please create one from the button next to the dropdown. Please contact your Administrator if you are unable to create one.
+
+Conditions:
+  IamPolicyEmpty: !Equals [!Ref IamPolicyDocument, '{}']
+
+Resources:
+  IAMRole:
+    Type: 'AWS::IAM::Role'
+    Properties:
+      RoleName: !Join ['-', [Ref: Namespace, 'jupyterLab-role']]
+      Path: '/'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: 'Allow'
+            Principal:
+              Service:
+                - 'ec2.amazonaws.com'
+            Action:
+              - 'sts:AssumeRole'
+      Policies:
+        - !If
+          - IamPolicyEmpty
+          - !Ref 'AWS::NoValue'
+          - PolicyName: !Join ['-', [Ref: Namespace, 's3-studydata-policy']]
+            PolicyDocument: !Ref IamPolicyDocument
+        - PolicyName: param-store-access
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: 'Allow'
+                Action:
+                  - 'ssm:GetParameter'
+                  - 'ssm:PutParameter'
+                Resource: !Sub 'arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/*'            
+
+  InstanceProfile:
+    Type: 'AWS::IAM::InstanceProfile'
+    Properties:
+      InstanceProfileName: !Join ['-', [Ref: Namespace, 'jupyterLab-profile']]
+      Path: '/'
+      Roles:
+        - Ref: IAMRole
+
+  EC2SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      SecurityGroupIngress:
+        - CidrIp: "0.0.0.0/0"
+          FromPort: 22
+          ToPort: 22
+          IpProtocol: tcp
+        - CidrIp: "0.0.0.0/0"
+          FromPort: 443
+          ToPort: 443
+          IpProtocol: tcp
+      GroupDescription: JupyterLab Security Group
+
+  EC2Instance:
+    Type: AWS::EC2::Instance
+    CreationPolicy:
+      ResourceSignal:
+        Timeout: PT10M
+    Metadata:
+      AWS::CloudFormation::Init:
+        configSets:
+          default:
+            - config1
+            - config2
+        config1:
+          files:
+            '/home/ec2-user/docker-compose.yml':
+              mode: '000755'
+              owner: 'ec2-user'
+              group: 'ec2-user'
+              content: !Sub |
+                version: "3.1"
+
+                services:
+                  nginx:
+                    container_name: nginx
+                    image: nginx:latest
+                    ports:
+                      - "80:80"
+                      - "443:443"  
+                    volumes:
+                      - /etc/nginx/nginx.conf:/etc/nginx/nginx.conf
+                      - /etc/nginx/dhparam.pem:/etc/nginx/dhparam.pem
+                      - /etc/nginx/cert.key:/etc/nginx/cert.key
+                      - /etc/nginx/cert.pem:/etc/nginx/cert.pem
+                    networks:
+                      - jupyterlab-net
+
+                  jupyterlab: 
+                    container_name: jupyterlab
+                    image: ${ContainerName}
+                    ports:
+                      - "8888:8888"
+                    volumes:
+                      - /home/ec2-user:/home/ec2-user
+                    networks:
+                      - jupyterlab-net
+
+                networks:
+                  jupyterlab-net:
+            '/home/ec2-user/mount_study.sh':
+              mode: '000755'
+              owner: 'ec2-user'
+              group: 'ec2-user'
+              content: !Sub |
+                #!/bin/sh
+                S3_MOUNTS='${S3Mounts}'
+                # Exit if no S3 mounts were specified
+                [ -z "$S3_MOUNTS" -o "$S3_MOUNTS" = "[]" ] && exit 0
+                su - ec2-user -c "env PATH=$PATH:/usr/local/bin mount_s3.sh" 2>&1 >> /home/ec2-user/mount_s3.log
+            '/home/ec2-user/start_jupyterlab.sh':
+              mode: '000755'
+              owner: 'ec2-user'
+              group: 'ec2-user'
+              content: !Sub |
+                #!/usr/bin/env bash
+                echo "fetching token and starting jupyterlab"
+                `docker-compose up -d`                           
+                sleep 5
+                instance_id=$(curl -s "http://169.254.169.254/latest/meta-data/instance-id")
+                instance_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region")
+                #access_token=$((docker exec jupyterlab  /bin/bash -c "jupyter server list" | grep token | awk '{print $1}') | sed 's/.*=//')
+                #echo "$access_token"
+                docker exec jupyterlab /bin/bash -c "jupyter server list" > access_token.txt 2>&1
+                sudo chmod 777 access_token.txt
+                access_token=$(grep -oP '(?<=token=)[^ ]+' access_token.txt)
+                echo "token:$access_token"
+                aws ssm put-parameter --name "/RL/RG/jupyterLab/auth-token/$instance_id" --value $access_token --region $instance_region --type SecureString --overwrite 
+                echo "Stored jupyterlab token in SSM"
+            '/var/log/jupyterlab.log':
+              content: "\n"
+              mode: '000664'
+              owner: 'ec2-user'
+              group: 'ec2-user'
+        config2:
+          commands:
+            01_mount_study:
+              cwd: '/home/ec2-user'
+              command: '/home/ec2-user/mount_study.sh'
+            02_start_jupyterlab:
+              cwd: '/home/ec2-user'
+              command: '/home/ec2-user/start_jupyterlab.sh'
+
+
+
+    Properties:
+      ImageId : '{{resolve:ssm:/RL/RG/StandardCatalog/JupyterLab}}'
+      InstanceType: !Ref 'InstanceType'
+      BlockDeviceMappings:
+        - DeviceName: /dev/xvda
+          Ebs:
+            VolumeSize: 32
+            Encrypted: false      
+      SecurityGroups: [!Ref 'EC2SecurityGroup']
+      KeyName: !Ref 'KeyPair'
+      IamInstanceProfile: !Ref InstanceProfile
+      PropagateTagsToVolumeOnCreation: true
+      Tags:
+        - Key: Name
+          Value: !Join ['-', [Ref: Namespace, 'jupyterLab']]
+        - Key: Description
+          Value: EC2 linux based JupyterLab server
+        - Key: cost_resource
+          Value: !Sub ${AWS::StackName}
+      UserData: 
+        Fn::Base64: !Sub |
+          #!/bin/bash 
+          exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+
+          # Install docker and pull necessary containers
+          sudo yum install -y docker
+          sudo systemctl enable docker.service
+          sudo systemctl enable containerd.service
+          sudo systemctl start docker
+          sudo usermod -a -G docker ec2-user
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose          
+          docker pull ${ContainerName}
+          docker pull nginx
+
+          # Copy environment instance files needed for the workspace
+          aws s3 cp --region "${AWS::Region}" "${EnvironmentInstanceFiles}/jupyterlab/" "/tmp/jupyterlab" --recursive
+          aws s3 cp --region "${AWS::Region}" "${EnvironmentInstanceFiles}/create_self_signed_cert.sh" "/tmp"
+          # aws s3 cp --region "${AWS::Region}" "${EnvironmentInstanceFiles}/secret.txt" "/root"
+
+          mkdir -p "/etc/nginx"
+          mv /tmp/jupyterlab/nginx.conf /etc/nginx/nginx.conf
+
+          # Create self signed certificates
+          chmod 500 "/tmp/create_self_signed_cert.sh"
+          /tmp/create_self_signed_cert.sh         
+
+          # Add command to start jupyterLab at reboot
+          sudo crontab -l 2>/dev/null > "/tmp/crontab"
+          echo '@reboot cd /home/ec2-user && sudo /home/ec2-user/mount_study.sh 2>&1 >> /home/ec2-user/mount_s3.log' >> "/tmp/crontab"
+          echo '@reboot cd /home/ec2-user && sudo /home/ec2-user/start_jupyterlab.sh 2>&1 >> /var/log/jupyterLab.log' >> "/tmp/crontab"          
+          sudo crontab "/tmp/crontab"
+
+          # Mount study
+          aws s3 cp --region "${AWS::Region}" "${EnvironmentInstanceFiles}/get_bootstrap.sh" "/tmp"
+          chmod 500 "/tmp/get_bootstrap.sh"
+          /tmp/get_bootstrap.sh "${EnvironmentInstanceFiles}" '${S3Mounts}' "${AWS::Region}"          
+
+          # init command to create files from config
+          /opt/aws/bin/cfn-init --verbose --stack ${AWS::StackName} --resource EC2Instance --region ${AWS::Region}
+
+          # Signal result to CloudFormation
+          /opt/aws/bin/cfn-signal -e $? --stack "${AWS::StackName}" --resource "EC2Instance" --region "${AWS::Region}"
+
+
+Outputs:
+  InstanceDNSName:
+    Value: !GetAtt EC2Instance.PublicDnsName
+    Description: Public DNS Name
+  InstanceIPAddress: 
+    Value: !GetAtt EC2Instance.PublicIp
+  InstanceId:
+    Value: !Ref 'EC2Instance'
+  ApplicationPort:
+    Value: '443'
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created JupyterLab EC2Instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  

cft-templates/ec2-linux-docker-mysql.yml

@@ -108,7 +108,8 @@ Resources:
           # Install Mysql Shell
           sudo yum install mysql -y
           # Pull Mysql Image
-          docker pull mysql:8
+          #docker pull mysql:8
+          docker pull mariadb:latest
           mkdir docker
           mkdir docker/mysql
           mkdir docker/mysql/conf
@@ -118,7 +119,7 @@ Resources:
           -v /home/ec2-user/docker/mysql/data/:/var/lib/mysql \
           -e MYSQL_ROOT_PASSWORD=${MasterPassword} \
           --restart always \
-          --name docker_mysql mysql:8 \
+          --name docker_mysql mariadb:latest \
           # Download and execute shell script
           cd /home/ec2-user
           aws s3 cp "${EnvironmentInstanceFiles}/alter_rootpassword.sh" "alter_rootpassword.sh"