From a4e6e3f44eb25643f9a6ceb4749d1d2db1a5810b Mon Sep 17 00:00:00 2001 From: Arran Ubels Date: Mon, 2 Jul 2018 12:40:46 +1000 Subject: [PATCH 01/11] I like my desktop notifications --- Gemfile | 3 ++- README.md | 1 + Rakefile | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index c8780d14..d14b3e20 100644 --- a/Gemfile +++ b/Gemfile @@ -6,4 +6,5 @@ gem 'cfn_manage' gem 'deep_merge' gem 'rubyzip' gem 'aws-sdk-s3', '~>1' -gem 'aws-sdk-cloudformation', '~>1' \ No newline at end of file +gem 'aws-sdk-cloudformation', '~>1' +gem 'notifier' diff --git a/README.md b/README.md index 749b945f..4c00fb48 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ update |Updates the ciinabox environment update_cert_to_acm |Replace previously auto-generated IAM certificate with auto-validated ACM certificate (if one exists) upload_server_cert |Uploads SSL server certs for ciinabox watch |Monitors status of the active ciinabox until failed or successful +watch_notify |Monitors status of the active ciinabox until failed or successful and sends messages to desktop via notifier ``` diff --git a/Rakefile b/Rakefile index 12c3521f..86c7b7ff 100644 --- a/Rakefile +++ b/Rakefile @@ -14,6 +14,7 @@ require_relative './ext/zip_helper' require 'aws-sdk-s3' require 'aws-sdk-cloudformation' require 'ciinabox-ecs' if Gem::Specification::find_all_by_name('ciinabox-ecs').any? +require 'notifier' namespace :ciinabox do @@ -258,6 +259,51 @@ namespace :ciinabox do end end + desc('Watches the status of the active ciinabox and sends a desktop notification message') + task :watch_notify do + last_status = "" + while true + check_active_ciinabox(config) + status, result = aws_execute(config, ['cloudformation', 'describe-stacks', "--stack-name #{stack_name}", '--query "Stacks[0].StackStatus"', '--out text']) + if status != 0 + puts "fail to get status for #{config['ciinabox_name']}...has it been created?" + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "fail to get status for #{config['ciinabox_name']}...has it been created?" + ) + exit 1 + end + output = result.chop! + next if last_status == output + if output == 'CREATE_COMPLETE' || output == 'UPDATE_COMPLETE' + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "ciinabox is alive!!!!" + ) + puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox is alive!!!!" + display_ecs_ip_address config + exit 0 + elsif output == 'ROLLBACK_IN_PROGRESS' + puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox has failed is being rolledback" + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "ciinabox has failed is being rolledback" + ) + elsif output == 'ROLLBACK_COMPLETE' + puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} rollbck completed" + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "rollbck completed" + ) + exit 1 + else + puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox is in state: #{output}" + end + last_status = output + sleep(4) + end + end + desc('Create self-signed SSL certs for use with ciinabox') task :create_server_cert do check_active_ciinabox(config) From 09a9687677e52ad892d9c66023948643bd6ecac8 Mon Sep 17 00:00:00 2001 From: Arran Ubels Date: Mon, 2 Jul 2018 12:47:05 +1000 Subject: [PATCH 02/11] Additional state checking --- Rakefile | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Rakefile b/Rakefile index 86c7b7ff..54ce3b88 100644 --- a/Rakefile +++ b/Rakefile @@ -266,11 +266,19 @@ namespace :ciinabox do check_active_ciinabox(config) status, result = aws_execute(config, ['cloudformation', 'describe-stacks', "--stack-name #{stack_name}", '--query "Stacks[0].StackStatus"', '--out text']) if status != 0 - puts "fail to get status for #{config['ciinabox_name']}...has it been created?" - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "fail to get status for #{config['ciinabox_name']}...has it been created?" - ) + if last_status == "" + puts "fail to get status for #{config['ciinabox_name']}...has it been created?" + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "fail to get status for #{config['ciinabox_name']}...has it been created?" + ) + else + puts "fail to get status for #{config['ciinabox_name']} disappeared from listing" + Notifier.notify( + title: "ciinabox-ecs: #{config['ciinabox_name']}", + message: "fail to get status for #{config['ciinabox_name']} disappeared from listing" + ) + end exit 1 end output = result.chop! From f2d4ffe03d82087b17f65af08813b6feb197b4f3 Mon Sep 17 00:00:00 2001 From: Guslington Date: Mon, 27 Sep 2021 13:32:03 +1000 Subject: [PATCH 03/11] add support for aws ip refix lists --- README.md | 17 +++++++++++++++++ templates/ecs-services.rb | 19 +++++++++++-------- templates/vpc.rb | 10 ++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 938a6e86..a491de03 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,23 @@ A common update would be to lock down ip access to your ciinabox environment .... ``` + or using AWS IP Prefix Lists + + ```yaml + .... + #Environment Access + #add list of public IP addresses you want to access the environment from + #default to public access probably best to change this + opsIpPrefixLists: + - pl-12345 + - pl-abcde + #add list of public IP addresses for your developers to access the environment + #default to public access probably best to change this + devAccess: + - pl-11111 + .... + ``` + 2. update your ciinabox ```bash $ ciinabox-ecs generate deploy update [ciinabox_name] diff --git a/templates/ecs-services.rb b/templates/ecs-services.rb index 0a34cb0e..21b890ca 100644 --- a/templates/ecs-services.rb +++ b/templates/ecs-services.rb @@ -137,20 +137,23 @@ ]) } - if defined? webHooks - rules = [] - webHooks.each do |ip| - rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', CidrIp: ip } - end - else - rules = [{ IpProtocol: 'tcp', FromPort: '443', ToPort: '443', CidrIp: '192.168.1.1/32' }] + webHooks = webHooks || [] + webHooksIpPrefixLists = webHooksIpPrefixLists || [] + + rules = [] + webHooks.each do |ip| + rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', CidrIp: ip } + end + + webHooksIpPrefixLists.each do |list| + rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', SourcePrefixListId: list } end Resource("SecurityGroupWebHooks") { Type 'AWS::EC2::SecurityGroup' Property('VpcId', Ref('VPC')) Property('GroupDescription', 'WebHooks like github') - Property('SecurityGroupIngress', rules) + Property('SecurityGroupIngress', rules) if rules.any? } Resource('ToolsSSLCertificate') { diff --git a/templates/vpc.rb b/templates/vpc.rb index fb0838a4..4eeffac8 100644 --- a/templates/vpc.rb +++ b/templates/vpc.rb @@ -167,6 +167,11 @@ rules << { IpProtocol: 'tcp', FromPort: '50000', ToPort: '50000', CidrIp: ip } end + opsIpPrefixLists.each do |list| + rules << { IpProtocol: 'tcp', FromPort: '80', ToPort: '80', SourcePrefixListId: list } + rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', SourcePrefixListId: list } + end + Resource("SecurityGroupOps") { Type 'AWS::EC2::SecurityGroup' Property('VpcId', Ref('VPC')) @@ -184,6 +189,11 @@ rules << { IpProtocol: 'tcp', FromPort: '50000', ToPort: '50000', CidrIp: ip } end + devIpPrefixLists.each do |list| + rules << { IpProtocol: 'tcp', FromPort: '80', ToPort: '80', SourcePrefixListId: list } + rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', SourcePrefixListId: list } + end + Resource("SecurityGroupDev") { Type 'AWS::EC2::SecurityGroup' Property('VpcId', Ref('VPC')) From a499b90754e13cf9042d906cc61c44c089302b8a Mon Sep 17 00:00:00 2001 From: Guslington Date: Mon, 27 Sep 2021 15:11:14 +1000 Subject: [PATCH 04/11] send ecs logs for nginx and jenkins to cloudwatch logs --- templates/ecs-services.rb | 16 ++++++++++++++++ templates/services/jenkins.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/templates/ecs-services.rb b/templates/ecs-services.rb index 0a34cb0e..91ad396a 100644 --- a/templates/ecs-services.rb +++ b/templates/ecs-services.rb @@ -261,6 +261,14 @@ end end + log_group_retention = log_group_retention || 90 + + Resource("LogGroup") { + Type "AWS::Logs::LogGroup" + Property("LogGroupName", "/ciinabox/#{ciinabox_name}/proxy") + Property("RetentionInDays", log_group_retention) + } + volumes = [] mount_points = [] @@ -290,6 +298,14 @@ HostPort: 8080, ContainerPort: 80 }], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "proxy" + } + }, Essential: true, MountPoints: mount_points } diff --git a/templates/services/jenkins.rb b/templates/services/jenkins.rb index f8a9d749..342c7aec 100644 --- a/templates/services/jenkins.rb +++ b/templates/services/jenkins.rb @@ -88,6 +88,14 @@ Value: '8080' } ], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins" + } + }, Essential: true, MountPoints: [ { @@ -112,6 +120,14 @@ Memory: slave_memory, Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 1}], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins-docker-dind-slave" + } + }, Essential: false, Privileged: true } @@ -143,6 +159,14 @@ Memory: slave_memory, Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 0}], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins-docker-dood-slave" + } + }, MountPoints: [ { ContainerPath: '/var/run/docker.sock', @@ -185,6 +209,14 @@ Parameter("ServiceELB") {Type 'String'} Parameter('InternalELB') {Type 'String'} if internal_elb + log_group_retention = log_group_retention || 90 + + Resource("LogGroup") { + Type "AWS::Logs::LogGroup" + Property("LogGroupName", "/ciinabox/#{ciinabox_name}/jenkins") + Property("RetentionInDays", log_group_retention) + } + Resource('JenkinsTask') { Type "AWS::ECS::TaskDefinition" Property('ContainerDefinitions', container_definitions) From 8e51eb6068ee5e56e39fa326c2a0998adce5e4fc Mon Sep 17 00:00:00 2001 From: Gus Vine Date: Mon, 27 Sep 2021 20:24:49 +1000 Subject: [PATCH 05/11] Revert "Like watch, but with desktop notifications" --- Gemfile | 3 +-- README.md | 1 - Rakefile | 54 ------------------------------------------------------ 3 files changed, 1 insertion(+), 57 deletions(-) diff --git a/Gemfile b/Gemfile index d14b3e20..c8780d14 100644 --- a/Gemfile +++ b/Gemfile @@ -6,5 +6,4 @@ gem 'cfn_manage' gem 'deep_merge' gem 'rubyzip' gem 'aws-sdk-s3', '~>1' -gem 'aws-sdk-cloudformation', '~>1' -gem 'notifier' +gem 'aws-sdk-cloudformation', '~>1' \ No newline at end of file diff --git a/README.md b/README.md index ba8cb30e..a491de03 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,6 @@ update |Updates the ciinabox environment update_cert_to_acm |Replace previously auto-generated IAM certificate with auto-validated ACM certificate (if one exists) upload_server_cert |Uploads SSL server certs for ciinabox watch |Monitors status of the active ciinabox until failed or successful -watch_notify |Monitors status of the active ciinabox until failed or successful and sends messages to desktop via notifier ``` diff --git a/Rakefile b/Rakefile index 984d0d8e..faf30e62 100644 --- a/Rakefile +++ b/Rakefile @@ -14,7 +14,6 @@ require_relative './ext/zip_helper' require 'aws-sdk-s3' require 'aws-sdk-cloudformation' require 'ciinabox-ecs' if Gem::Specification::find_all_by_name('ciinabox-ecs').any? -require 'notifier' namespace :ciinabox do @@ -261,59 +260,6 @@ namespace :ciinabox do end end - desc('Watches the status of the active ciinabox and sends a desktop notification message') - task :watch_notify do - last_status = "" - while true - check_active_ciinabox(config) - status, result = aws_execute(config, ['cloudformation', 'describe-stacks', "--stack-name #{stack_name}", '--query "Stacks[0].StackStatus"', '--out text']) - if status != 0 - if last_status == "" - puts "fail to get status for #{config['ciinabox_name']}...has it been created?" - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "fail to get status for #{config['ciinabox_name']}...has it been created?" - ) - else - puts "fail to get status for #{config['ciinabox_name']} disappeared from listing" - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "fail to get status for #{config['ciinabox_name']} disappeared from listing" - ) - end - exit 1 - end - output = result.chop! - next if last_status == output - if output == 'CREATE_COMPLETE' || output == 'UPDATE_COMPLETE' - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "ciinabox is alive!!!!" - ) - puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox is alive!!!!" - display_ecs_ip_address config - exit 0 - elsif output == 'ROLLBACK_IN_PROGRESS' - puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox has failed is being rolledback" - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "ciinabox has failed is being rolledback" - ) - elsif output == 'ROLLBACK_COMPLETE' - puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} rollbck completed" - Notifier.notify( - title: "ciinabox-ecs: #{config['ciinabox_name']}", - message: "rollbck completed" - ) - exit 1 - else - puts Time.now.strftime("%Y/%m/%d %H:%M") + " #{config['ciinabox_name']} ciinabox is in state: #{output}" - end - last_status = output - sleep(4) - end - end - desc('Create self-signed SSL certs for use with ciinabox') task :create_server_cert do check_active_ciinabox(config) From cf76e6256213d72aa713d998875c524c022063b0 Mon Sep 17 00:00:00 2001 From: Kyle Thorne Date: Tue, 28 Sep 2021 12:36:18 +1000 Subject: [PATCH 06/11] Fixed up readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a491de03..44a6c89d 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,7 @@ A common update would be to lock down ip access to your ciinabox environment or using AWS IP Prefix Lists - ```yaml + ```yaml .... #Environment Access #add list of public IP addresses you want to access the environment from @@ -246,7 +246,7 @@ A common update would be to lock down ip access to your ciinabox environment - pl-abcde #add list of public IP addresses for your developers to access the environment #default to public access probably best to change this - devAccess: + devIpPrefixLists: - pl-11111 .... ``` From f4dd38c82cc800ae78b762b9e238c8d0e0a5992a Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 28 Sep 2021 13:35:55 +1000 Subject: [PATCH 07/11] set default values for ip prefix lists if not set --- templates/vpc.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/vpc.rb b/templates/vpc.rb index 4eeffac8..59a1f753 100644 --- a/templates/vpc.rb +++ b/templates/vpc.rb @@ -167,6 +167,8 @@ rules << { IpProtocol: 'tcp', FromPort: '50000', ToPort: '50000', CidrIp: ip } end + opsIpPrefixLists = opsIpPrefixLists || [] + opsIpPrefixLists.each do |list| rules << { IpProtocol: 'tcp', FromPort: '80', ToPort: '80', SourcePrefixListId: list } rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', SourcePrefixListId: list } @@ -189,6 +191,8 @@ rules << { IpProtocol: 'tcp', FromPort: '50000', ToPort: '50000', CidrIp: ip } end + devIpPrefixLists = devIpPrefixLists || [] + devIpPrefixLists.each do |list| rules << { IpProtocol: 'tcp', FromPort: '80', ToPort: '80', SourcePrefixListId: list } rules << { IpProtocol: 'tcp', FromPort: '443', ToPort: '443', SourcePrefixListId: list } From 70e2cae24716808a69bb1520e160e6e6c9fb769f Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 28 Sep 2021 13:36:18 +1000 Subject: [PATCH 08/11] put whole template in cfndsl context --- templates/services/jenkins.rb | 376 +++++++++++++++++----------------- 1 file changed, 187 insertions(+), 189 deletions(-) diff --git a/templates/services/jenkins.rb b/templates/services/jenkins.rb index 342c7aec..8812c802 100644 --- a/templates/services/jenkins.rb +++ b/templates/services/jenkins.rb @@ -1,213 +1,211 @@ require 'cfndsl' require_relative '../../ext/helper' -if !defined? timezone - timezone = 'GMT' -end - -if !defined? internal_elb - internal_elb = nil -end - -if !defined? volatile_jenkins_slave - volatile_jenkins_slave = false -end - -# Prefixing application images allows us to 'vendorize' ciinabox into client's account by setting -# ciinabox_repo to ${account_no}.dkr.ecr.${region}.amazonaws.com -if not defined? ciinabox_repo - ciinabox_repo = 'ghcr.io/base2services' -end -image = "#{ciinabox_repo}/ciinabox-jenkins:lts" - -jenkins_java_opts = '' -memory = 2048 -slave_memory = 2048 -cpu = 300 -container_port = 0 -service = lookup_service('jenkins', services) -virtual_host = "jenkins.#{dns_domain}" -if defined? internal_elb and internal_elb - virtual_host = "#{virtual_host},internal-jenkins.#{dns_domain}" -end -port_mappings = [] - -if defined? service - service = {} if service.nil? - jenkins_java_opts = service['JAVA_OPTS'] || '' - image = service['ContainerImage'] || image - memory = service['ContainerMemory'] || 2048 - slave_memory = service['SlaveContainerMemory'] || 2048 - cpu = service['ContainerCPU'] || 300 - - if service['InstancePort'] - port_mappings << { - HostPort: service['InstancePort'], - ContainerPort: service['InstancePort'] - } - container_port = service['InstancePort'] - virtual_host = "jenkins.#{dns_domain},internal-jenkins.#{dns_domain}" +CloudFormation { + AWSTemplateFormatVersion "2010-09-09" + Description "ciinabox - ECS Service Jenkins v#{ciinabox_version}" + + Parameter("ECSCluster") {Type 'String'} + Parameter("ECSRole") {Type 'String'} + Parameter("ServiceELB") {Type 'String'} + Parameter('InternalELB') {Type 'String'} if internal_elb + + if !defined? timezone + timezone = 'GMT' end -end + if !defined? internal_elb + internal_elb = nil + end -# container volumes and container definitions depending on feature flags -volumes = [ - { - Name: 'timezone', - Host: { - SourcePath: '/etc/localtime' - } - }, - { - Name: 'jenkins_data', - Host: { - SourcePath: '/data/jenkins' - } - }] - -container_definitions = [ - { - Name: 'jenkins', - Links: [], - Memory: memory, - Cpu: cpu, - Image: image, - PortMappings: port_mappings, - Environment: [ - { - Name: 'JAVA_OPTS', - Value: "#{jenkins_java_opts} -Duser.timezone=#{timezone}" - }, - { - Name: 'VIRTUAL_HOST', - Value: virtual_host - }, - { - Name: 'VIRTUAL_PORT', - Value: '8080' - } - ], + if !defined? volatile_jenkins_slave + volatile_jenkins_slave = false + end + + # Prefixing application images allows us to 'vendorize' ciinabox into client's account by setting + # ciinabox_repo to ${account_no}.dkr.ecr.${region}.amazonaws.com + if not defined? ciinabox_repo + ciinabox_repo = 'ghcr.io/base2services' + end + image = "#{ciinabox_repo}/ciinabox-jenkins:lts" + + jenkins_java_opts = '' + memory = 2048 + slave_memory = 2048 + cpu = 300 + container_port = 0 + service = lookup_service('jenkins', services) + virtual_host = "jenkins.#{dns_domain}" + if defined? internal_elb and internal_elb + virtual_host = "#{virtual_host},internal-jenkins.#{dns_domain}" + end + port_mappings = [] + + if defined? service + service = {} if service.nil? + jenkins_java_opts = service['JAVA_OPTS'] || '' + image = service['ContainerImage'] || image + memory = service['ContainerMemory'] || 2048 + slave_memory = service['SlaveContainerMemory'] || 2048 + cpu = service['ContainerCPU'] || 300 + + if service['InstancePort'] + port_mappings << { + HostPort: service['InstancePort'], + ContainerPort: service['InstancePort'] + } + container_port = service['InstancePort'] + virtual_host = "jenkins.#{dns_domain},internal-jenkins.#{dns_domain}" + end + + end + + # container volumes and container definitions depending on feature flags + volumes = [ + { + Name: 'timezone', + Host: { + SourcePath: '/etc/localtime' + } + }, + { + Name: 'jenkins_data', + Host: { + SourcePath: '/data/jenkins' + } + }] + + container_definitions = [ + { + Name: 'jenkins', + Links: [], + Memory: memory, + Cpu: cpu, + Image: image, + PortMappings: port_mappings, + Environment: [ + { + Name: 'JAVA_OPTS', + Value: "#{jenkins_java_opts} -Duser.timezone=#{timezone}" + }, + { + Name: 'VIRTUAL_HOST', + Value: virtual_host + }, + { + Name: 'VIRTUAL_PORT', + Value: '8080' + } + ], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins" + } + }, + Essential: true, + MountPoints: [ + { + ContainerPath: '/etc/localtime', + SourceVolume: 'timezone', + ReadOnly: true + }, + { + ContainerPath: '/var/jenkins_home', + SourceVolume: 'jenkins_data', + ReadOnly: false + } + ] + } + ] + + # If docker in docker slave is enabled + if defined? include_diind_slave and include_diind_slave + container_definitions[0][:Links] << 'jenkins-docker-dind-slave' + dind_definition = { + Name: 'jenkins-docker-dind-slave', + Memory: slave_memory, + Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", + Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 1}], LogConfiguration: { - LogDriver: 'awslogs', - Options: { - 'awslogs-group' => Ref("LogGroup"), - "awslogs-region" => Ref("AWS::Region"), - "awslogs-stream-prefix" => "jenkins" - } + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins-docker-dind-slave" + } + }, + Essential: false, + Privileged: true + } + dind_definition[:Environment] << { Name: 'USE_ECR_CREDENTIAL_HELPER', Value: 1 } if docker_slave_enable_ecr_credentials_helper + if not volatile_jenkins_slave + dind_definition[:MountPoints] = [ + { + ContainerPath: '/var/lib/docker', + SourceVolume: 'jenkins_dind_data', + ReadOnly: false + } + ] + volumes << { + Name: 'jenkins_dind_data', + Host: { + SourcePath: '/data/jenkins-diind' + } + } + end + container_definitions << dind_definition + + end + + # If docker outside of docker slave is enabled + if defined? include_dood_slave and include_dood_slave + container_definitions[0][:Links] << 'jenkins-docker-dood-slave' + dood_definition = { + Name: 'jenkins-docker-dood-slave', + Memory: slave_memory, + Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", + Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 0}], + LogConfiguration: { + LogDriver: 'awslogs', + Options: { + 'awslogs-group' => Ref("LogGroup"), + "awslogs-region" => Ref("AWS::Region"), + "awslogs-stream-prefix" => "jenkins-docker-dood-slave" + } }, - Essential: true, MountPoints: [ { - ContainerPath: '/etc/localtime', - SourceVolume: 'timezone', - ReadOnly: true + ContainerPath: '/var/run/docker.sock', + SourceVolume: 'docker_socket', + ReadOnly: false }, { - ContainerPath: '/var/jenkins_home', - SourceVolume: 'jenkins_data', + ContainerPath: '/data/jenkins-dood', + SourceVolume: 'jenkins_dood_data', ReadOnly: false } - ] + ], + Essential: false, + Privileged: false } -] - -# If docker in docker slave is enabled -if defined? include_diind_slave and include_diind_slave - container_definitions[0][:Links] << 'jenkins-docker-dind-slave' - dind_definition = { - Name: 'jenkins-docker-dind-slave', - Memory: slave_memory, - Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", - Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 1}], - LogConfiguration: { - LogDriver: 'awslogs', - Options: { - 'awslogs-group' => Ref("LogGroup"), - "awslogs-region" => Ref("AWS::Region"), - "awslogs-stream-prefix" => "jenkins-docker-dind-slave" - } - }, - Essential: false, - Privileged: true - } - dind_definition[:Environment] << { Name: 'USE_ECR_CREDENTIAL_HELPER', Value: 1 } if docker_slave_enable_ecr_credentials_helper - if not volatile_jenkins_slave - dind_definition[:MountPoints] = [ - { - ContainerPath: '/var/lib/docker', - SourceVolume: 'jenkins_dind_data', - ReadOnly: false + dood_definition[:Environment] << { Name: 'USE_ECR_CREDENTIAL_HELPER', Value: 1 } if docker_slave_enable_ecr_credentials_helper + container_definitions << dood_definition + volumes << { + Name: 'jenkins_dood_data', + Host: { + SourcePath: '/data/jenkins-dood' } - ] + } volumes << { - Name: 'jenkins_dind_data', + Name: 'docker_socket', Host: { - SourcePath: '/data/jenkins-diind' + SourcePath: '/var/run/docker.sock' } } end - container_definitions << dind_definition - -end - -# If docker outside of docker slave is enabled -if defined? include_dood_slave and include_dood_slave - container_definitions[0][:Links] << 'jenkins-docker-dood-slave' - dood_definition = { - Name: 'jenkins-docker-dood-slave', - Memory: slave_memory, - Image: "#{ciinabox_repo}/ciinabox-docker-slave:#{docker_slave_version}", - Environment: [{Name: 'RUN_DOCKER_IN_DOCKER', Value: 0}], - LogConfiguration: { - LogDriver: 'awslogs', - Options: { - 'awslogs-group' => Ref("LogGroup"), - "awslogs-region" => Ref("AWS::Region"), - "awslogs-stream-prefix" => "jenkins-docker-dood-slave" - } - }, - MountPoints: [ - { - ContainerPath: '/var/run/docker.sock', - SourceVolume: 'docker_socket', - ReadOnly: false - }, - { - ContainerPath: '/data/jenkins-dood', - SourceVolume: 'jenkins_dood_data', - ReadOnly: false - } - ], - Essential: false, - Privileged: false - } - dood_definition[:Environment] << { Name: 'USE_ECR_CREDENTIAL_HELPER', Value: 1 } if docker_slave_enable_ecr_credentials_helper - container_definitions << dood_definition - volumes << { - Name: 'jenkins_dood_data', - Host: { - SourcePath: '/data/jenkins-dood' - } - } - volumes << { - Name: 'docker_socket', - Host: { - SourcePath: '/var/run/docker.sock' - } - } -end - - -CloudFormation { - - AWSTemplateFormatVersion "2010-09-09" - Description "ciinabox - ECS Service Jenkins v#{ciinabox_version}" - - Parameter("ECSCluster") {Type 'String'} - Parameter("ECSRole") {Type 'String'} - Parameter("ServiceELB") {Type 'String'} - Parameter('InternalELB') {Type 'String'} if internal_elb log_group_retention = log_group_retention || 90 From af004085547d4052a40f0851c19a68e079053ceb Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 28 Sep 2021 13:36:49 +1000 Subject: [PATCH 09/11] replace ci with github actions --- .github/.github/workflows/build-gem.yml | 25 +++++++++++ .github/.github/workflows/release-gem.yml | 34 +++++++++++++++ .github/.github/workflows/release-image.yml | 33 +++++++++++++++ .travis.yml | 46 --------------------- Dockerfile | 6 +-- README.md | 2 +- ciinabox-ecs.gemspec | 2 +- 7 files changed, 97 insertions(+), 51 deletions(-) create mode 100644 .github/.github/workflows/build-gem.yml create mode 100644 .github/.github/workflows/release-gem.yml create mode 100644 .github/.github/workflows/release-image.yml delete mode 100644 .travis.yml diff --git a/.github/.github/workflows/build-gem.yml b/.github/.github/workflows/build-gem.yml new file mode 100644 index 00000000..60b6a41e --- /dev/null +++ b/.github/.github/workflows/build-gem.yml @@ -0,0 +1,25 @@ +name: test and build gem +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + name: test + build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: set up ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7.x + - name: rspec + run: | + gem install rspec + rspec + - name: build gem + run: | + gem build ciinabox-ecs.gemspec \ No newline at end of file diff --git a/.github/.github/workflows/release-gem.yml b/.github/.github/workflows/release-gem.yml new file mode 100644 index 00000000..b9fac216 --- /dev/null +++ b/.github/.github/workflows/release-gem.yml @@ -0,0 +1,34 @@ +name: release gem + +on: + release: + types: [published] + +jobs: + build: + name: Build + Publish Gem + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7.x + + - name: rspec + run: | + gem install rspec + rspec + + - name: build gem + run: | + gem build ciinabox-ecs.gemspec + + - name: Publish gem + uses: dawidd6/action-publish-gem@v1 + with: + api_key: ${{secrets.RUBYGEMS_API_KEY}} + github_token: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file diff --git a/.github/.github/workflows/release-image.yml b/.github/.github/workflows/release-image.yml new file mode 100644 index 00000000..461b44b2 --- /dev/null +++ b/.github/.github/workflows/release-image.yml @@ -0,0 +1,33 @@ +name: release docker image + +on: + release: + types: [published] + +jobs: + build: + name: Build + Publish Container Image + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to GitHub Container Repository + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Container Image to GitHub Container Repository + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/ciinabox-ecs:${{ github.event.release.tag_name }} + build-args: CIINABOX_ECS_VERSION=${{ github.event.release.tag_name }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b0db5836..00000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -sudo: required -dist: trusty -rvm: - - 2.3 -python: - - 3.6 -#before_install: -# - | -# sudo apt-get update && \ -# sudo apt-get install software-properties-common -y && \ -# sudo add-apt-repository ppa:deadsnakes/ppa -y && \ -# sudo apt-add-repository ppa:brightbox/ruby-ng -y && \ -# sudo apt-get update && sudo apt-get install python3.6 python3-pip -y && \ -# sudo apt-get install ruby2.3 -y -script: - - gem build ciinabox-ecs.gemspec - - gem install ciinabox-ecs-*.gem - - which ciinabox-ecs && ciinabox-ecs help - - cfndsl -u 9.0.0 - - | - git clone https://github.com/base2services/ciinabox-ecs-examples - cd ciinabox-ecs-examples - which pip - git checkout master - set -x - for ciinabox in demo_* ; do - printf "\n\nTesting ${ciinabox}\n\n" - # avoid validation in PRs as aws creds are not available - set +e - if [[ "$TRAVIS_EVENT_TYPE" =~ ^push|api$ ]]; then - ciinabox-ecs generate validate ${ciinabox} - else - ciinabox-ecs generate ${ciinabox} - fi - if [ $? -ne 0 ]; then - printf "\n\nCIINABOX test configuration ${ciinabox} failed\n\n" - exit 2 - fi - done - - cd .. -deploy: - provider: rubygems - api_key: "${RUBYGEMS_API_KEY}" - on: - all_branches: true - condition: $TRAVIS_BRANCH =~ ^develop|master && $TRAVIS_EVENT_TYPE =~ ^push|api$ && $TRAVIS_REPO_SLUG == "base2Services/ciinabox-ecs" diff --git a/Dockerfile b/Dockerfile index 948b30c0..d0c0e211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ FROM ruby:2.5-alpine ARG CFNDSL_SPEC_VERSION=${CFNDSL_SPEC_VERSION:-9.0.0} +ARG CIINABOX_ECS_VERSION='*' COPY . /src WORKDIR /src -RUN rm ciinabox-ecs-*.gem ; \ - gem build ciinabox-ecs.gemspec && \ - gem install ciinabox-ecs-*.gem && \ +RUN gem build ciinabox-ecs.gemspec && \ + gem install ciinabox-ecs-${CIINABOX_ECS_VERSION}.gem && \ rm -rf /src RUN adduser -u 1000 -D ciinabox && \ diff --git a/README.md b/README.md index a491de03..0a6f78b3 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ A common update would be to lock down ip access to your ciinabox environment - pl-abcde #add list of public IP addresses for your developers to access the environment #default to public access probably best to change this - devAccess: + devIpPrefixLists: - pl-11111 .... ``` diff --git a/ciinabox-ecs.gemspec b/ciinabox-ecs.gemspec index 320bd06f..bbb32b20 100644 --- a/ciinabox-ecs.gemspec +++ b/ciinabox-ecs.gemspec @@ -3,7 +3,7 @@ require 'date' Gem::Specification.new do |s| s.name = 'ciinabox-ecs' - s.version = '0.3.2' + s.version = '0.4.0' s.version = "#{s.version}.alpha.#{Time.now.getutc.to_i}" if ENV['TRAVIS'] and ENV['TRAVIS_BRANCH'] != 'master' s.date = Date.today.to_s s.summary = 'Manage ciinabox on Aws Ecs' From 7835091a37ebe18fb9148710e3238f7ca155aac8 Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 28 Sep 2021 13:44:08 +1000 Subject: [PATCH 10/11] correct folder for actions --- .github/{.github => }/workflows/build-gem.yml | 7 ++++--- .github/{.github => }/workflows/release-gem.yml | 0 .github/{.github => }/workflows/release-image.yml | 0 3 files changed, 4 insertions(+), 3 deletions(-) rename .github/{.github => }/workflows/build-gem.yml (85%) rename .github/{.github => }/workflows/release-gem.yml (100%) rename .github/{.github => }/workflows/release-image.yml (100%) diff --git a/.github/.github/workflows/build-gem.yml b/.github/workflows/build-gem.yml similarity index 85% rename from .github/.github/workflows/build-gem.yml rename to .github/workflows/build-gem.yml index 60b6a41e..75036357 100644 --- a/.github/.github/workflows/build-gem.yml +++ b/.github/workflows/build-gem.yml @@ -1,9 +1,10 @@ name: test and build gem on: push: - branches: [ master ] - pull_request: - branches: [ master ] + branches: + - master + - develop + - feature/* jobs: build: diff --git a/.github/.github/workflows/release-gem.yml b/.github/workflows/release-gem.yml similarity index 100% rename from .github/.github/workflows/release-gem.yml rename to .github/workflows/release-gem.yml diff --git a/.github/.github/workflows/release-image.yml b/.github/workflows/release-image.yml similarity index 100% rename from .github/.github/workflows/release-image.yml rename to .github/workflows/release-image.yml From 763ed87a781e16f94c0703fafd66b1a307c91986 Mon Sep 17 00:00:00 2001 From: Guslington Date: Tue, 28 Sep 2021 13:44:59 +1000 Subject: [PATCH 11/11] add source label to dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d0c0e211..e288b0bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM ruby:2.5-alpine +LABEL org.opencontainers.image.source = https://github.com/base2Services/ciinabox-ecs + ARG CFNDSL_SPEC_VERSION=${CFNDSL_SPEC_VERSION:-9.0.0} ARG CIINABOX_ECS_VERSION='*'