From f39a47041d1b2cb51e3711c3b3794c3d60f9b316 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 7 Aug 2019 06:53:24 -0700 Subject: [PATCH 01/61] added mgmt EIP --- BIG-IP/template.json | 6 ++++++ deprovision.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100755 deprovision.sh diff --git a/BIG-IP/template.json b/BIG-IP/template.json index 6872641..435fa08 100644 --- a/BIG-IP/template.json +++ b/BIG-IP/template.json @@ -319,6 +319,9 @@ "Description": "IP address of the management port on BIG-IP", "Value": { "Ref": "Bigip1ManagementEipAddress" + }, + "Export" : { + "Name" : {"Fn::Sub": "${AWS::StackName}-Bigip1ManagementEipAddress" } } }, "Bigip1ManagementInterface": { @@ -434,6 +437,9 @@ "Description": "IP address of the management port on BIG-IP", "Value": { "Ref": "Bigip2ManagementEipAddress" + }, + "Export" : { + "Name" : {"Fn::Sub": "${AWS::StackName}-Bigip2ManagementEipAddress" } } }, "Bigip2ManagementInterface": { diff --git a/deprovision.sh b/deprovision.sh new file mode 100755 index 0000000..9dc4e47 --- /dev/null +++ b/deprovision.sh @@ -0,0 +1,27 @@ +# delete S3 bucket content for BIG-IP clusters +# aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackName')].[Name]" +for i in `aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackName')].[Name]" | jq -r .[][][]`; +do + echo $i + aws s3 ls s3://$i + aws s3 rm s3://$i --recursive +done + +# find CFT outputs + + +# find BIG-IP management IP addresses +# aws cloudformation list-exports --query 'Exports[?contains(Name, `cody-sca-test`)]' +# aws cloudformation list-exports --query 'Exports[?contains(Name, `cody-sca-test`)]|[?contains(Name, `BIGIP`)]|[?contains(Name, `External`)].[Value]' +# aws ec2 describe-network-interfaces --network-interface-id=eni-03bd3f82d34f9eed4 +# aws ec2 describe-network-interfaces --network-interface-id=eni-03bd3f82d34f9eed4 --query 'NetworkInterfaces[*].Association.[PublicIp]' + +# release BIG-IP licenses +# for i in {1..4} +# do +# echo "bigip$i"; +# # echo $bigip1 +# ip=bigip${i} +# echo "${!ip}" +# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"${!ip}" 'modify cli preference pager disabled display-threshold 0; revoke sys license' +# done \ No newline at end of file From 34bdad45e093005691610ece4070d98096a68b23 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 7 Aug 2019 09:41:43 -0700 Subject: [PATCH 02/61] updated order of BIG-IP revoke license --- deprovision.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/deprovision.sh b/deprovision.sh index 9dc4e47..6914818 100755 --- a/deprovision.sh +++ b/deprovision.sh @@ -1,5 +1,6 @@ +if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi + # delete S3 bucket content for BIG-IP clusters -# aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackName')].[Name]" for i in `aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackName')].[Name]" | jq -r .[][][]`; do echo $i @@ -7,21 +8,20 @@ do aws s3 rm s3://$i --recursive done -# find CFT outputs - +# find BIG-IP management IP addresses, deprovision internal stacks before external stacks +for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP2')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +do + echo "revoke license for $ip" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" 'modify cli preference pager disabled display-threshold 0; revoke sys license' +done -# find BIG-IP management IP addresses -# aws cloudformation list-exports --query 'Exports[?contains(Name, `cody-sca-test`)]' -# aws cloudformation list-exports --query 'Exports[?contains(Name, `cody-sca-test`)]|[?contains(Name, `BIGIP`)]|[?contains(Name, `External`)].[Value]' -# aws ec2 describe-network-interfaces --network-interface-id=eni-03bd3f82d34f9eed4 -# aws ec2 describe-network-interfaces --network-interface-id=eni-03bd3f82d34f9eed4 --query 'NetworkInterfaces[*].Association.[PublicIp]' +# deprovision external stack +for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +do + echo "revoke license for $ip" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" 'modify cli preference pager disabled display-threshold 0; revoke sys license' +done -# release BIG-IP licenses -# for i in {1..4} -# do -# echo "bigip$i"; -# # echo $bigip1 -# ip=bigip${i} -# echo "${!ip}" -# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"${!ip}" 'modify cli preference pager disabled display-threshold 0; revoke sys license' -# done \ No newline at end of file +# delete CFT stack +echo "deleting CFT stack $stackName" +aws cloudformation delete-stack --stack-name $stackName \ No newline at end of file From 5bb0e687ab4afde92e90022e1c366b8e32fb5e4d Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 7 Aug 2019 11:15:44 -0700 Subject: [PATCH 03/61] abstracted deploy variables that were hard coded --- .gitignore | 1 + deploy.sh | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 393046e..9b63e7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +.DS_Store load-keys.sh diff --git a/deploy.sh b/deploy.sh index d6227c0..7019d15 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,12 +1,14 @@ +if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi +if [ -z ${sshKeyName} ]; then echo "stackName is unset" && exit 1; fi if [ -z ${licenseKey1} ]; then echo "licenseKey1 is unset" && exit 1; fi if [ -z ${licenseKey2} ]; then echo "licenseKey2 is unset" && exit 1; fi if [ -z ${licenseKey3} ]; then echo "licenseKey3 is unset" && exit 1; fi if [ -z ${licenseKey4} ]; then echo "licenseKey4 is unset" && exit 1; fi +if [ -z ${S3Key} ]; then export s3Key="master"; fi aws cloudformation create-stack \ ---stack-name cody-sca-test \ +--stack-name $stackName \ --template-body file://./aws-quickstart-scca-main-same-net.json \ --disable-rollback \ --capabilities CAPABILITY_IAM \ ---parameters ParameterKey=licenseKey1,ParameterValue=$licenseKey1 ParameterKey=licenseKey2,ParameterValue=$licenseKey2 ParameterKey=licenseKey3,ParameterValue=$licenseKey3 ParameterKey=licenseKey4,ParameterValue=$licenseKey4 ParameterKey=pBaselineCompliance,ParameterValue="Enterprise" ParameterKey=pQuickstartS3KeyPrefix,ParameterValue="sca-cody" ParameterKey=sshKey,ParameterValue="cody-key" - +--parameters ParameterKey=licenseKey1,ParameterValue=$licenseKey1 ParameterKey=licenseKey2,ParameterValue=$licenseKey2 ParameterKey=licenseKey3,ParameterValue=$licenseKey3 ParameterKey=licenseKey4,ParameterValue=$licenseKey4 ParameterKey=pBaselineCompliance,ParameterValue="Enterprise" ParameterKey=pQuickstartS3KeyPrefix,ParameterValue="$S3Key" ParameterKey=sshKey,ParameterValue="$sshKeyName" From b0b7c476551d9a42d6366169d199709cd67d0055 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 7 Aug 2019 17:53:13 -0700 Subject: [PATCH 04/61] added ha failover iapp --- BIG-IP/template.json | 10 ++++++++-- test.sh | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 test.sh diff --git a/BIG-IP/template.json b/BIG-IP/template.json index 435fa08..95670f6 100644 --- a/BIG-IP/template.json +++ b/BIG-IP/template.json @@ -1117,6 +1117,12 @@ "mode": "000755", "owner": "root" }, + "/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl": { + "group": "root", + "mode": "000755", + "owner": "root", + "source": "http://cdn.f5.com/product/cloudsolutions/iapps/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl" + }, "/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl": { "group": "root", "mode": "000755", @@ -1259,7 +1265,7 @@ " exit", "fi", "echo loaded verifyHash", - "declare -a filesToVerify=(\"/config/cloud/f5-cloud-libs.tar.gz\" \"/config/cloud/f5-cloud-libs-aws.tar.gz\" \"/config/cloud/f5-appsvcs-3.5.1-5.noarch.rpm\" \"/config/cloud/aws/f5.service_discovery.tmpl\" \"/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl\")", + "declare -a filesToVerify=(\"/config/cloud/f5-cloud-libs.tar.gz\" \"/config/cloud/f5-cloud-libs-aws.tar.gz\" \"/config/cloud/f5-appsvcs-3.5.1-5.noarch.rpm\" \"/config/cloud/aws/f5.service_discovery.tmpl\" \"/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl\" \"/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl\")", "for fileToVerify in \"${filesToVerify[@]}\"", "do", " echo verifying \"$fileToVerify\"", @@ -1284,7 +1290,7 @@ "owner": "root" }, "/config/verifyHash": { - "content": "cli script /Common/verifyHash {\nproc script::run {} {\n if {[catch {\n set hashes(f5-cloud-libs.tar.gz) 86d24521063640bfaaafa882f491974e39b46c700554dd8591d681139df25b0bed4dfe934545c2894e0a27a513ae7498a52e6deac00b5399f301badbfb8a91e0\n set hashes(f5-cloud-libs-aws.tar.gz) 076c969cbfff12efacce0879820262b7787c98645f1105667cc4927d4acfe2466ed64c777b6d35957f6df7ae266937dde42fef4c8b1f870020a366f7f910ffb5\n set hashes(f5-cloud-libs-azure.tar.gz) 57fae388e8aa028d24a2d3fa2c029776925011a72edb320da47ccd4fb8dc762321c371312f692b7b8f1c84e8261c280f6887ba2e0f841b50547e6e6abc8043ba\n set hashes(f5-cloud-libs-gce.tar.gz) 1677835e69967fd9882ead03cbdd24b426627133b8db9e41f6de5a26fef99c2d7b695978ac189f00f61c0737e6dbb638d42dea43a867ef4c01d9507d0ee1fb2f\n set hashes(f5-cloud-libs-openstack.tar.gz) 5c83fe6a93a6fceb5a2e8437b5ed8cc9faf4c1621bfc9e6a0779f6c2137b45eab8ae0e7ed745c8cf821b9371245ca29749ca0b7e5663949d77496b8728f4b0f9\n set hashes(f5-cloud-libs-consul.tar.gz) a32aab397073df92cbbba5067e5823e9b5fafca862a258b60b6b40aa0975c3989d1e110f706177b2ffbe4dde65305a260a5856594ce7ad4ef0c47b694ae4a513\n set hashes(asm-policy-linux.tar.gz) 63b5c2a51ca09c43bd89af3773bbab87c71a6e7f6ad9410b229b4e0a1c483d46f1a9fff39d9944041b02ee9260724027414de592e99f4c2475415323e18a72e0\n set hashes(f5.http.v1.2.0rc4.tmpl) 47c19a83ebfc7bd1e9e9c35f3424945ef8694aa437eedd17b6a387788d4db1396fefe445199b497064d76967b0d50238154190ca0bd73941298fc257df4dc034\n set hashes(f5.http.v1.2.0rc6.tmpl) 811b14bffaab5ed0365f0106bb5ce5e4ec22385655ea3ac04de2a39bd9944f51e3714619dae7ca43662c956b5212228858f0592672a2579d4a87769186e2cbfe\n set hashes(f5.http.v1.2.0rc7.tmpl) 21f413342e9a7a281a0f0e1301e745aa86af21a697d2e6fdc21dd279734936631e92f34bf1c2d2504c201f56ccd75c5c13baa2fe7653213689ec3c9e27dff77d\n set hashes(f5.aws_advanced_ha.v1.3.0rc1.tmpl) 9e55149c010c1d395abdae3c3d2cb83ec13d31ed39424695e88680cf3ed5a013d626b326711d3d40ef2df46b72d414b4cb8e4f445ea0738dcbd25c4c843ac39d\n set hashes(f5.aws_advanced_ha.v1.4.0rc1.tmpl) de068455257412a949f1eadccaee8506347e04fd69bfb645001b76f200127668e4a06be2bbb94e10fefc215cfc3665b07945e6d733cbe1a4fa1b88e881590396\n set hashes(f5.aws_advanced_ha.v1.4.0rc2.tmpl) 6ab0bffc426df7d31913f9a474b1a07860435e366b07d77b32064acfb2952c1f207beaed77013a15e44d80d74f3253e7cf9fbbe12a90ec7128de6facd097d68f\n set hashes(f5.aws_advanced_ha.v1.4.0rc3.tmpl) 2f2339b4bc3a23c9cfd42aae2a6de39ba0658366f25985de2ea53410a745f0f18eedc491b20f4a8dba8db48970096e2efdca7b8efffa1a83a78e5aadf218b134\n set hashes(f5.aws_advanced_ha.v1.4.0rc4.tmpl) 2418ac8b1f1884c5c096cbac6a94d4059aaaf05927a6a4508fd1f25b8cc6077498839fbdda8176d2cf2d274a27e6a1dae2a1e3a0a9991bc65fc74fc0d02ce963\n set hashes(f5.aws_advanced_ha.v1.4.0rc5.tmpl) 5e582187ae1a6323e095d41eddd41151d6bd38eb83c634410d4527a3d0e246a8fc62685ab0849de2ade62b0275f51264d2deaccbc16b773417f847a4a1ea9bc4\n set hashes(asm-policy.tar.gz) 2d39ec60d006d05d8a1567a1d8aae722419e8b062ad77d6d9a31652971e5e67bc4043d81671ba2a8b12dd229ea46d205144f75374ed4cae58cefa8f9ab6533e6\n set hashes(deploy_waf.sh) 1a3a3c6274ab08a7dc2cb73aedc8d2b2a23cd9e0eb06a2e1534b3632f250f1d897056f219d5b35d3eed1207026e89989f754840fd92969c515ae4d829214fb74\n set hashes(f5.policy_creator.tmpl) 06539e08d115efafe55aa507ecb4e443e83bdb1f5825a9514954ef6ca56d240ed00c7b5d67bd8f67b815ee9dd46451984701d058c89dae2434c89715d375a620\n set hashes(f5.service_discovery.tmpl) 4811a95372d1dbdbb4f62f8bcc48d4bc919fa492cda012c81e3a2fe63d7966cc36ba8677ed049a814a930473234f300d3f8bced2b0db63176d52ac99640ce81b\n set hashes(f5.cloud_logger.v1.0.0.tmpl) 64a0ed3b5e32a037ba4e71d460385fe8b5e1aecc27dc0e8514b511863952e419a89f4a2a43326abb543bba9bc34376afa114ceda950d2c3bd08dab735ff5ad20\n set hashes(f5-appsvcs-3.5.1-5.noarch.rpm) ba71c6e1c52d0c7077cdb25a58709b8fb7c37b34418a8338bbf67668339676d208c1a4fef4e5470c152aac84020b4ccb8074ce387de24be339711256c0fa78c8\n\n set file_path [lindex $tmsh::argv 1]\n set file_name [file tail $file_path]\n\n if {![info exists hashes($file_name)]} {\n tmsh::log err \"No hash found for $file_name\"\n exit 1\n }\n\n set expected_hash $hashes($file_name)\n set computed_hash [lindex [exec /usr/bin/openssl dgst -r -sha512 $file_path] 0]\n if { $expected_hash eq $computed_hash } {\n exit 0\n }\n tmsh::log err \"Hash does not match for $file_path\"\n exit 1\n }]} {\n tmsh::log err {Unexpected error in verifyHash}\n exit 1\n }\n }\n script-signature TjGIQtBj6gmevtbkCb9uwhrzm02aFer0KYwVeIvwBPssYRAJ7T/mc91LWODrSs9U12XAAtxQKHsG3XYWvklyB79fe/7C+589WvIhG814/OoXd1dplcH7O+EF5LBMdXEY4VQNw+5HuSl89tvBIkHGMCXSRAj8hHDIx9kqOF4OirExaJBZopb2qTY0P7AmhjpzFCvkJLUcmz7H5LZCfFCDYWWB5g5Bst7ZkLWfP2arMtxLRTFl/m6x4QttOeD9wktPGajn1XhvXThtzJ8vlTJJRuk/OyLJPr+64qkXwi3wngudiiZeFnCiI1Mrggp96B+yKtCqRsABJju6LMdH80T1Zg==\n signing-key /Common/f5-irule\n}", + "content": "cli script /Common/verifyHash {\nproc script::run {} {\n if {[catch {\n set hashes(f5-cloud-libs.tar.gz) 86d24521063640bfaaafa882f491974e39b46c700554dd8591d681139df25b0bed4dfe934545c2894e0a27a513ae7498a52e6deac00b5399f301badbfb8a91e0\n set hashes(f5-cloud-libs-aws.tar.gz) 076c969cbfff12efacce0879820262b7787c98645f1105667cc4927d4acfe2466ed64c777b6d35957f6df7ae266937dde42fef4c8b1f870020a366f7f910ffb5\n set hashes(f5-cloud-libs-azure.tar.gz) 57fae388e8aa028d24a2d3fa2c029776925011a72edb320da47ccd4fb8dc762321c371312f692b7b8f1c84e8261c280f6887ba2e0f841b50547e6e6abc8043ba\n set hashes(f5-cloud-libs-gce.tar.gz) 1677835e69967fd9882ead03cbdd24b426627133b8db9e41f6de5a26fef99c2d7b695978ac189f00f61c0737e6dbb638d42dea43a867ef4c01d9507d0ee1fb2f\n set hashes(f5-cloud-libs-openstack.tar.gz) 5c83fe6a93a6fceb5a2e8437b5ed8cc9faf4c1621bfc9e6a0779f6c2137b45eab8ae0e7ed745c8cf821b9371245ca29749ca0b7e5663949d77496b8728f4b0f9\n set hashes(f5-cloud-libs-consul.tar.gz) a32aab397073df92cbbba5067e5823e9b5fafca862a258b60b6b40aa0975c3989d1e110f706177b2ffbe4dde65305a260a5856594ce7ad4ef0c47b694ae4a513\n set hashes(asm-policy-linux.tar.gz) 63b5c2a51ca09c43bd89af3773bbab87c71a6e7f6ad9410b229b4e0a1c483d46f1a9fff39d9944041b02ee9260724027414de592e99f4c2475415323e18a72e0\n set hashes(f5.http.v1.2.0rc4.tmpl) 47c19a83ebfc7bd1e9e9c35f3424945ef8694aa437eedd17b6a387788d4db1396fefe445199b497064d76967b0d50238154190ca0bd73941298fc257df4dc034\n set hashes(f5.http.v1.2.0rc6.tmpl) 811b14bffaab5ed0365f0106bb5ce5e4ec22385655ea3ac04de2a39bd9944f51e3714619dae7ca43662c956b5212228858f0592672a2579d4a87769186e2cbfe\n set hashes(f5.http.v1.2.0rc7.tmpl) 21f413342e9a7a281a0f0e1301e745aa86af21a697d2e6fdc21dd279734936631e92f34bf1c2d2504c201f56ccd75c5c13baa2fe7653213689ec3c9e27dff77d\n set hashes(f5.aws_advanced_ha.v1.3.0rc1.tmpl) 9e55149c010c1d395abdae3c3d2cb83ec13d31ed39424695e88680cf3ed5a013d626b326711d3d40ef2df46b72d414b4cb8e4f445ea0738dcbd25c4c843ac39d\n set hashes(f5.aws_advanced_ha.v1.4.0rc1.tmpl) de068455257412a949f1eadccaee8506347e04fd69bfb645001b76f200127668e4a06be2bbb94e10fefc215cfc3665b07945e6d733cbe1a4fa1b88e881590396\n set hashes(f5.aws_advanced_ha.v1.4.0rc2.tmpl) 6ab0bffc426df7d31913f9a474b1a07860435e366b07d77b32064acfb2952c1f207beaed77013a15e44d80d74f3253e7cf9fbbe12a90ec7128de6facd097d68f\n set hashes(f5.aws_advanced_ha.v1.4.0rc3.tmpl) 2f2339b4bc3a23c9cfd42aae2a6de39ba0658366f25985de2ea53410a745f0f18eedc491b20f4a8dba8db48970096e2efdca7b8efffa1a83a78e5aadf218b134\n set hashes(f5.aws_advanced_ha.v1.4.0rc4.tmpl) 2418ac8b1f1884c5c096cbac6a94d4059aaaf05927a6a4508fd1f25b8cc6077498839fbdda8176d2cf2d274a27e6a1dae2a1e3a0a9991bc65fc74fc0d02ce963\n set hashes(asm-policy.tar.gz) 2d39ec60d006d05d8a1567a1d8aae722419e8b062ad77d6d9a31652971e5e67bc4043d81671ba2a8b12dd229ea46d205144f75374ed4cae58cefa8f9ab6533e6\n set hashes(deploy_waf.sh) 1a3a3c6274ab08a7dc2cb73aedc8d2b2a23cd9e0eb06a2e1534b3632f250f1d897056f219d5b35d3eed1207026e89989f754840fd92969c515ae4d829214fb74\n set hashes(f5.policy_creator.tmpl) 06539e08d115efafe55aa507ecb4e443e83bdb1f5825a9514954ef6ca56d240ed00c7b5d67bd8f67b815ee9dd46451984701d058c89dae2434c89715d375a620\n set hashes(f5.service_discovery.tmpl) 4811a95372d1dbdbb4f62f8bcc48d4bc919fa492cda012c81e3a2fe63d7966cc36ba8677ed049a814a930473234f300d3f8bced2b0db63176d52ac99640ce81b\n set hashes(f5.cloud_logger.v1.0.0.tmpl) 64a0ed3b5e32a037ba4e71d460385fe8b5e1aecc27dc0e8514b511863952e419a89f4a2a43326abb543bba9bc34376afa114ceda950d2c3bd08dab735ff5ad20\n set hashes(f5-appsvcs-3.5.1-5.noarch.rpm) ba71c6e1c52d0c7077cdb25a58709b8fb7c37b34418a8338bbf67668339676d208c1a4fef4e5470c152aac84020b4ccb8074ce387de24be339711256c0fa78c8\n\n set hashes(f5.aws_advanced_ha.v1.4.0rc5.tmpl) 5e582187ae1a6323e095d41eddd41151d6bd38eb83c634410d4527a3d0e246a8fc62685ab0849de2ade62b0275f51264d2deaccbc16b773417f847a4a1ea9bc4\n set file_path [lindex $tmsh::argv 1]\n set file_name [file tail $file_path]\n\n if {![info exists hashes($file_name)]} {\n tmsh::log err \"No hash found for $file_name\"\n exit 1\n }\n\n set expected_hash $hashes($file_name)\n set computed_hash [lindex [exec /usr/bin/openssl dgst -r -sha512 $file_path] 0]\n if { $expected_hash eq $computed_hash } {\n exit 0\n }\n tmsh::log err \"Hash does not match for $file_path\"\n exit 1\n }]} {\n tmsh::log err {Unexpected error in verifyHash}\n exit 1\n }\n }\n script-signature TjGIQtBj6gmevtbkCb9uwhrzm02aFer0KYwVeIvwBPssYRAJ7T/mc91LWODrSs9U12XAAtxQKHsG3XYWvklyB79fe/7C+589WvIhG814/OoXd1dplcH7O+EF5LBMdXEY4VQNw+5HuSl89tvBIkHGMCXSRAj8hHDIx9kqOF4OirExaJBZopb2qTY0P7AmhjpzFCvkJLUcmz7H5LZCfFCDYWWB5g5Bst7ZkLWfP2arMtxLRTFl/m6x4QttOeD9wktPGajn1XhvXThtzJ8vlTJJRuk/OyLJPr+64qkXwi3wngudiiZeFnCiI1Mrggp96B+yKtCqRsABJju6LMdH80T1Zg==\n signing-key /Common/f5-irule\n}", "group": "root", "mode": "000755", "owner": "root" diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..830e96c --- /dev/null +++ b/test.sh @@ -0,0 +1,21 @@ +if [ -z ${BIGIP_PWD} ]; then echo "stackName is unset" && exit 1; fi +# Change admin password on BIG-IP appliances in tier-1 +for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +do + echo "set password for Tier-1 BIG-IP $ip" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" +done + +# Change admin password on BIG-IP appliances in tier-2 +for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP2')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +do + echo "set password for Tier-2 BIG-IP $ip" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" +done + +# install ha iApp and configure +for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +do + echo "install HA iApp for Tier-1 BIG-IP $ip" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl" +done From 0f40ea1886f9f5344c53574fac1aed76626c4ddf Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 7 Aug 2019 21:08:58 -0700 Subject: [PATCH 05/61] revert to upstream --- BIG-IP/template.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/BIG-IP/template.json b/BIG-IP/template.json index 95670f6..435fa08 100644 --- a/BIG-IP/template.json +++ b/BIG-IP/template.json @@ -1117,12 +1117,6 @@ "mode": "000755", "owner": "root" }, - "/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl": { - "group": "root", - "mode": "000755", - "owner": "root", - "source": "http://cdn.f5.com/product/cloudsolutions/iapps/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl" - }, "/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl": { "group": "root", "mode": "000755", @@ -1265,7 +1259,7 @@ " exit", "fi", "echo loaded verifyHash", - "declare -a filesToVerify=(\"/config/cloud/f5-cloud-libs.tar.gz\" \"/config/cloud/f5-cloud-libs-aws.tar.gz\" \"/config/cloud/f5-appsvcs-3.5.1-5.noarch.rpm\" \"/config/cloud/aws/f5.service_discovery.tmpl\" \"/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl\" \"/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl\")", + "declare -a filesToVerify=(\"/config/cloud/f5-cloud-libs.tar.gz\" \"/config/cloud/f5-cloud-libs-aws.tar.gz\" \"/config/cloud/f5-appsvcs-3.5.1-5.noarch.rpm\" \"/config/cloud/aws/f5.service_discovery.tmpl\" \"/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl\")", "for fileToVerify in \"${filesToVerify[@]}\"", "do", " echo verifying \"$fileToVerify\"", @@ -1290,7 +1284,7 @@ "owner": "root" }, "/config/verifyHash": { - "content": "cli script /Common/verifyHash {\nproc script::run {} {\n if {[catch {\n set hashes(f5-cloud-libs.tar.gz) 86d24521063640bfaaafa882f491974e39b46c700554dd8591d681139df25b0bed4dfe934545c2894e0a27a513ae7498a52e6deac00b5399f301badbfb8a91e0\n set hashes(f5-cloud-libs-aws.tar.gz) 076c969cbfff12efacce0879820262b7787c98645f1105667cc4927d4acfe2466ed64c777b6d35957f6df7ae266937dde42fef4c8b1f870020a366f7f910ffb5\n set hashes(f5-cloud-libs-azure.tar.gz) 57fae388e8aa028d24a2d3fa2c029776925011a72edb320da47ccd4fb8dc762321c371312f692b7b8f1c84e8261c280f6887ba2e0f841b50547e6e6abc8043ba\n set hashes(f5-cloud-libs-gce.tar.gz) 1677835e69967fd9882ead03cbdd24b426627133b8db9e41f6de5a26fef99c2d7b695978ac189f00f61c0737e6dbb638d42dea43a867ef4c01d9507d0ee1fb2f\n set hashes(f5-cloud-libs-openstack.tar.gz) 5c83fe6a93a6fceb5a2e8437b5ed8cc9faf4c1621bfc9e6a0779f6c2137b45eab8ae0e7ed745c8cf821b9371245ca29749ca0b7e5663949d77496b8728f4b0f9\n set hashes(f5-cloud-libs-consul.tar.gz) a32aab397073df92cbbba5067e5823e9b5fafca862a258b60b6b40aa0975c3989d1e110f706177b2ffbe4dde65305a260a5856594ce7ad4ef0c47b694ae4a513\n set hashes(asm-policy-linux.tar.gz) 63b5c2a51ca09c43bd89af3773bbab87c71a6e7f6ad9410b229b4e0a1c483d46f1a9fff39d9944041b02ee9260724027414de592e99f4c2475415323e18a72e0\n set hashes(f5.http.v1.2.0rc4.tmpl) 47c19a83ebfc7bd1e9e9c35f3424945ef8694aa437eedd17b6a387788d4db1396fefe445199b497064d76967b0d50238154190ca0bd73941298fc257df4dc034\n set hashes(f5.http.v1.2.0rc6.tmpl) 811b14bffaab5ed0365f0106bb5ce5e4ec22385655ea3ac04de2a39bd9944f51e3714619dae7ca43662c956b5212228858f0592672a2579d4a87769186e2cbfe\n set hashes(f5.http.v1.2.0rc7.tmpl) 21f413342e9a7a281a0f0e1301e745aa86af21a697d2e6fdc21dd279734936631e92f34bf1c2d2504c201f56ccd75c5c13baa2fe7653213689ec3c9e27dff77d\n set hashes(f5.aws_advanced_ha.v1.3.0rc1.tmpl) 9e55149c010c1d395abdae3c3d2cb83ec13d31ed39424695e88680cf3ed5a013d626b326711d3d40ef2df46b72d414b4cb8e4f445ea0738dcbd25c4c843ac39d\n set hashes(f5.aws_advanced_ha.v1.4.0rc1.tmpl) de068455257412a949f1eadccaee8506347e04fd69bfb645001b76f200127668e4a06be2bbb94e10fefc215cfc3665b07945e6d733cbe1a4fa1b88e881590396\n set hashes(f5.aws_advanced_ha.v1.4.0rc2.tmpl) 6ab0bffc426df7d31913f9a474b1a07860435e366b07d77b32064acfb2952c1f207beaed77013a15e44d80d74f3253e7cf9fbbe12a90ec7128de6facd097d68f\n set hashes(f5.aws_advanced_ha.v1.4.0rc3.tmpl) 2f2339b4bc3a23c9cfd42aae2a6de39ba0658366f25985de2ea53410a745f0f18eedc491b20f4a8dba8db48970096e2efdca7b8efffa1a83a78e5aadf218b134\n set hashes(f5.aws_advanced_ha.v1.4.0rc4.tmpl) 2418ac8b1f1884c5c096cbac6a94d4059aaaf05927a6a4508fd1f25b8cc6077498839fbdda8176d2cf2d274a27e6a1dae2a1e3a0a9991bc65fc74fc0d02ce963\n set hashes(asm-policy.tar.gz) 2d39ec60d006d05d8a1567a1d8aae722419e8b062ad77d6d9a31652971e5e67bc4043d81671ba2a8b12dd229ea46d205144f75374ed4cae58cefa8f9ab6533e6\n set hashes(deploy_waf.sh) 1a3a3c6274ab08a7dc2cb73aedc8d2b2a23cd9e0eb06a2e1534b3632f250f1d897056f219d5b35d3eed1207026e89989f754840fd92969c515ae4d829214fb74\n set hashes(f5.policy_creator.tmpl) 06539e08d115efafe55aa507ecb4e443e83bdb1f5825a9514954ef6ca56d240ed00c7b5d67bd8f67b815ee9dd46451984701d058c89dae2434c89715d375a620\n set hashes(f5.service_discovery.tmpl) 4811a95372d1dbdbb4f62f8bcc48d4bc919fa492cda012c81e3a2fe63d7966cc36ba8677ed049a814a930473234f300d3f8bced2b0db63176d52ac99640ce81b\n set hashes(f5.cloud_logger.v1.0.0.tmpl) 64a0ed3b5e32a037ba4e71d460385fe8b5e1aecc27dc0e8514b511863952e419a89f4a2a43326abb543bba9bc34376afa114ceda950d2c3bd08dab735ff5ad20\n set hashes(f5-appsvcs-3.5.1-5.noarch.rpm) ba71c6e1c52d0c7077cdb25a58709b8fb7c37b34418a8338bbf67668339676d208c1a4fef4e5470c152aac84020b4ccb8074ce387de24be339711256c0fa78c8\n\n set hashes(f5.aws_advanced_ha.v1.4.0rc5.tmpl) 5e582187ae1a6323e095d41eddd41151d6bd38eb83c634410d4527a3d0e246a8fc62685ab0849de2ade62b0275f51264d2deaccbc16b773417f847a4a1ea9bc4\n set file_path [lindex $tmsh::argv 1]\n set file_name [file tail $file_path]\n\n if {![info exists hashes($file_name)]} {\n tmsh::log err \"No hash found for $file_name\"\n exit 1\n }\n\n set expected_hash $hashes($file_name)\n set computed_hash [lindex [exec /usr/bin/openssl dgst -r -sha512 $file_path] 0]\n if { $expected_hash eq $computed_hash } {\n exit 0\n }\n tmsh::log err \"Hash does not match for $file_path\"\n exit 1\n }]} {\n tmsh::log err {Unexpected error in verifyHash}\n exit 1\n }\n }\n script-signature TjGIQtBj6gmevtbkCb9uwhrzm02aFer0KYwVeIvwBPssYRAJ7T/mc91LWODrSs9U12XAAtxQKHsG3XYWvklyB79fe/7C+589WvIhG814/OoXd1dplcH7O+EF5LBMdXEY4VQNw+5HuSl89tvBIkHGMCXSRAj8hHDIx9kqOF4OirExaJBZopb2qTY0P7AmhjpzFCvkJLUcmz7H5LZCfFCDYWWB5g5Bst7ZkLWfP2arMtxLRTFl/m6x4QttOeD9wktPGajn1XhvXThtzJ8vlTJJRuk/OyLJPr+64qkXwi3wngudiiZeFnCiI1Mrggp96B+yKtCqRsABJju6LMdH80T1Zg==\n signing-key /Common/f5-irule\n}", + "content": "cli script /Common/verifyHash {\nproc script::run {} {\n if {[catch {\n set hashes(f5-cloud-libs.tar.gz) 86d24521063640bfaaafa882f491974e39b46c700554dd8591d681139df25b0bed4dfe934545c2894e0a27a513ae7498a52e6deac00b5399f301badbfb8a91e0\n set hashes(f5-cloud-libs-aws.tar.gz) 076c969cbfff12efacce0879820262b7787c98645f1105667cc4927d4acfe2466ed64c777b6d35957f6df7ae266937dde42fef4c8b1f870020a366f7f910ffb5\n set hashes(f5-cloud-libs-azure.tar.gz) 57fae388e8aa028d24a2d3fa2c029776925011a72edb320da47ccd4fb8dc762321c371312f692b7b8f1c84e8261c280f6887ba2e0f841b50547e6e6abc8043ba\n set hashes(f5-cloud-libs-gce.tar.gz) 1677835e69967fd9882ead03cbdd24b426627133b8db9e41f6de5a26fef99c2d7b695978ac189f00f61c0737e6dbb638d42dea43a867ef4c01d9507d0ee1fb2f\n set hashes(f5-cloud-libs-openstack.tar.gz) 5c83fe6a93a6fceb5a2e8437b5ed8cc9faf4c1621bfc9e6a0779f6c2137b45eab8ae0e7ed745c8cf821b9371245ca29749ca0b7e5663949d77496b8728f4b0f9\n set hashes(f5-cloud-libs-consul.tar.gz) a32aab397073df92cbbba5067e5823e9b5fafca862a258b60b6b40aa0975c3989d1e110f706177b2ffbe4dde65305a260a5856594ce7ad4ef0c47b694ae4a513\n set hashes(asm-policy-linux.tar.gz) 63b5c2a51ca09c43bd89af3773bbab87c71a6e7f6ad9410b229b4e0a1c483d46f1a9fff39d9944041b02ee9260724027414de592e99f4c2475415323e18a72e0\n set hashes(f5.http.v1.2.0rc4.tmpl) 47c19a83ebfc7bd1e9e9c35f3424945ef8694aa437eedd17b6a387788d4db1396fefe445199b497064d76967b0d50238154190ca0bd73941298fc257df4dc034\n set hashes(f5.http.v1.2.0rc6.tmpl) 811b14bffaab5ed0365f0106bb5ce5e4ec22385655ea3ac04de2a39bd9944f51e3714619dae7ca43662c956b5212228858f0592672a2579d4a87769186e2cbfe\n set hashes(f5.http.v1.2.0rc7.tmpl) 21f413342e9a7a281a0f0e1301e745aa86af21a697d2e6fdc21dd279734936631e92f34bf1c2d2504c201f56ccd75c5c13baa2fe7653213689ec3c9e27dff77d\n set hashes(f5.aws_advanced_ha.v1.3.0rc1.tmpl) 9e55149c010c1d395abdae3c3d2cb83ec13d31ed39424695e88680cf3ed5a013d626b326711d3d40ef2df46b72d414b4cb8e4f445ea0738dcbd25c4c843ac39d\n set hashes(f5.aws_advanced_ha.v1.4.0rc1.tmpl) de068455257412a949f1eadccaee8506347e04fd69bfb645001b76f200127668e4a06be2bbb94e10fefc215cfc3665b07945e6d733cbe1a4fa1b88e881590396\n set hashes(f5.aws_advanced_ha.v1.4.0rc2.tmpl) 6ab0bffc426df7d31913f9a474b1a07860435e366b07d77b32064acfb2952c1f207beaed77013a15e44d80d74f3253e7cf9fbbe12a90ec7128de6facd097d68f\n set hashes(f5.aws_advanced_ha.v1.4.0rc3.tmpl) 2f2339b4bc3a23c9cfd42aae2a6de39ba0658366f25985de2ea53410a745f0f18eedc491b20f4a8dba8db48970096e2efdca7b8efffa1a83a78e5aadf218b134\n set hashes(f5.aws_advanced_ha.v1.4.0rc4.tmpl) 2418ac8b1f1884c5c096cbac6a94d4059aaaf05927a6a4508fd1f25b8cc6077498839fbdda8176d2cf2d274a27e6a1dae2a1e3a0a9991bc65fc74fc0d02ce963\n set hashes(f5.aws_advanced_ha.v1.4.0rc5.tmpl) 5e582187ae1a6323e095d41eddd41151d6bd38eb83c634410d4527a3d0e246a8fc62685ab0849de2ade62b0275f51264d2deaccbc16b773417f847a4a1ea9bc4\n set hashes(asm-policy.tar.gz) 2d39ec60d006d05d8a1567a1d8aae722419e8b062ad77d6d9a31652971e5e67bc4043d81671ba2a8b12dd229ea46d205144f75374ed4cae58cefa8f9ab6533e6\n set hashes(deploy_waf.sh) 1a3a3c6274ab08a7dc2cb73aedc8d2b2a23cd9e0eb06a2e1534b3632f250f1d897056f219d5b35d3eed1207026e89989f754840fd92969c515ae4d829214fb74\n set hashes(f5.policy_creator.tmpl) 06539e08d115efafe55aa507ecb4e443e83bdb1f5825a9514954ef6ca56d240ed00c7b5d67bd8f67b815ee9dd46451984701d058c89dae2434c89715d375a620\n set hashes(f5.service_discovery.tmpl) 4811a95372d1dbdbb4f62f8bcc48d4bc919fa492cda012c81e3a2fe63d7966cc36ba8677ed049a814a930473234f300d3f8bced2b0db63176d52ac99640ce81b\n set hashes(f5.cloud_logger.v1.0.0.tmpl) 64a0ed3b5e32a037ba4e71d460385fe8b5e1aecc27dc0e8514b511863952e419a89f4a2a43326abb543bba9bc34376afa114ceda950d2c3bd08dab735ff5ad20\n set hashes(f5-appsvcs-3.5.1-5.noarch.rpm) ba71c6e1c52d0c7077cdb25a58709b8fb7c37b34418a8338bbf67668339676d208c1a4fef4e5470c152aac84020b4ccb8074ce387de24be339711256c0fa78c8\n\n set file_path [lindex $tmsh::argv 1]\n set file_name [file tail $file_path]\n\n if {![info exists hashes($file_name)]} {\n tmsh::log err \"No hash found for $file_name\"\n exit 1\n }\n\n set expected_hash $hashes($file_name)\n set computed_hash [lindex [exec /usr/bin/openssl dgst -r -sha512 $file_path] 0]\n if { $expected_hash eq $computed_hash } {\n exit 0\n }\n tmsh::log err \"Hash does not match for $file_path\"\n exit 1\n }]} {\n tmsh::log err {Unexpected error in verifyHash}\n exit 1\n }\n }\n script-signature TjGIQtBj6gmevtbkCb9uwhrzm02aFer0KYwVeIvwBPssYRAJ7T/mc91LWODrSs9U12XAAtxQKHsG3XYWvklyB79fe/7C+589WvIhG814/OoXd1dplcH7O+EF5LBMdXEY4VQNw+5HuSl89tvBIkHGMCXSRAj8hHDIx9kqOF4OirExaJBZopb2qTY0P7AmhjpzFCvkJLUcmz7H5LZCfFCDYWWB5g5Bst7ZkLWfP2arMtxLRTFl/m6x4QttOeD9wktPGajn1XhvXThtzJ8vlTJJRuk/OyLJPr+64qkXwi3wngudiiZeFnCiI1Mrggp96B+yKtCqRsABJju6LMdH80T1Zg==\n signing-key /Common/f5-irule\n}", "group": "root", "mode": "000755", "owner": "root" From 776ac13e93a300fb21a1c6e507969447e1759686 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 8 Aug 2019 10:24:36 -0700 Subject: [PATCH 06/61] download ha iApp --- BIG-IP/template.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BIG-IP/template.json b/BIG-IP/template.json index 435fa08..6dcacf3 100644 --- a/BIG-IP/template.json +++ b/BIG-IP/template.json @@ -1117,6 +1117,12 @@ "mode": "000755", "owner": "root" }, + "/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl": { + "group": "root", + "mode": "000755", + "owner": "root", + "source": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl" + }, "/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl": { "group": "root", "mode": "000755", From beee245cf327d365061db9447676c427376e22b7 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 8 Aug 2019 11:04:03 -0700 Subject: [PATCH 07/61] load ha iapp --- BIG-IP/template.json | 7 +++++++ test.sh | 36 ++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/BIG-IP/template.json b/BIG-IP/template.json index 6dcacf3..d246b44 100644 --- a/BIG-IP/template.json +++ b/BIG-IP/template.json @@ -1071,6 +1071,7 @@ "\"tmsh modify cm device ${HOSTNAME} unicast-address { { effective-ip ${INTIP} effective-port 1026 ip ${INTIP} } }\"\n", "\"tmsh load sys application template /config/cloud/aws/f5.service_discovery.tmpl\"\n", "\"tmsh load sys application template /config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl\"\n", + "\"tmsh load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl\"\n", "\"tmsh save /sys config\")\n", "for CMD in \"${tmsh[@]}\"\n", "do\n", @@ -2185,6 +2186,12 @@ "mode": "000755", "owner": "root" }, + "/config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl": { + "group": "root", + "mode": "000755", + "owner": "root", + "source": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl" + }, "/config/cloud/aws/f5.cloud_logger.v1.0.0.tmpl": { "group": "root", "mode": "000755", diff --git a/test.sh b/test.sh index 830e96c..eec3514 100755 --- a/test.sh +++ b/test.sh @@ -1,21 +1,29 @@ -if [ -z ${BIGIP_PWD} ]; then echo "stackName is unset" && exit 1; fi +# if [ -z ${BIGIP_PWD} ]; then echo "stackName is unset" && exit 1; fi # Change admin password on BIG-IP appliances in tier-1 -for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; -do - echo "set password for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" -done +# for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +# do +# echo "set password for Tier-1 BIG-IP $ip" +# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" +# done + +# # Change admin password on BIG-IP appliances in tier-2 +# for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP2')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +# do +# echo "set password for Tier-2 BIG-IP $ip" +# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" +# done + +# Get password for BIG-IPs in tier-1 +if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi +S3bucket=`aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackName-f5bigip1')].[Name]" | jq -r .[][][]` +export bigip1_pwd=`aws s3 cp s3://$S3bucket/credentials/master - | jq -r .password` +echo $bigip1_pwd + -# Change admin password on BIG-IP appliances in tier-2 -for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP2')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; -do - echo "set password for Tier-2 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" -done -# install ha iApp and configure +# # install ha iApp and configure for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; do echo "install HA iApp for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc5.tmpl" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl" done From 1281c0b54e8c156f69215ab1cd94f2e7a22af99e Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 8 Aug 2019 12:00:01 -0700 Subject: [PATCH 08/61] configure iapp - step 1 --- test.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test.sh b/test.sh index eec3514..f159dd9 100755 --- a/test.sh +++ b/test.sh @@ -25,5 +25,14 @@ echo $bigip1_pwd for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; do echo "install HA iApp for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes } }" done + +# below line is from across AZ iApp -don't use, for notes only +#"create /sys application service HA_Across_AZs template f5.aws_advanced_ha.v1.4.0rc3 tables add { eip_mappings__mappings { column-names { eip az1_vip az2_vip } rows { { row { ${VIPEIP} /Common/${EXTPRIVIP} /Common/${PEER_EXTPRIVIP} } } } } } variables add { eip_mappings__inbound { value yes } }\"\n", + + + + + +#"load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl" \ No newline at end of file From 424d4941ad7a1c471fdae9b746a0f3a9977f3003 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 8 Aug 2019 12:35:25 -0700 Subject: [PATCH 09/61] fixed issue with iapp interface assignment --- test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.sh b/test.sh index f159dd9..0372ac0 100755 --- a/test.sh +++ b/test.sh @@ -25,13 +25,13 @@ echo $bigip1_pwd for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; do echo "install HA iApp for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes } }" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface (value "/Common/internal" ) }" done # below line is from across AZ iApp -don't use, for notes only #"create /sys application service HA_Across_AZs template f5.aws_advanced_ha.v1.4.0rc3 tables add { eip_mappings__mappings { column-names { eip az1_vip az2_vip } rows { { row { ${VIPEIP} /Common/${EXTPRIVIP} /Common/${PEER_EXTPRIVIP} } } } } } variables add { eip_mappings__inbound { value yes } }\"\n", - +create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } } From 104334c698c7a98a85ea612832fe3c4341c1f616 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Fri, 16 Aug 2019 11:08:12 -0500 Subject: [PATCH 10/61] configure HA iApp --- test.sh | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test.sh b/test.sh index 0372ac0..f6d9a7d 100755 --- a/test.sh +++ b/test.sh @@ -25,14 +25,5 @@ echo $bigip1_pwd for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; do echo "install HA iApp for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface (value "/Common/internal" ) }" + ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }" done - -# below line is from across AZ iApp -don't use, for notes only -#"create /sys application service HA_Across_AZs template f5.aws_advanced_ha.v1.4.0rc3 tables add { eip_mappings__mappings { column-names { eip az1_vip az2_vip } rows { { row { ${VIPEIP} /Common/${EXTPRIVIP} /Common/${PEER_EXTPRIVIP} } } } } } variables add { eip_mappings__inbound { value yes } }\"\n", - -create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } } - - - -#"load sys application template /config/cloud/aws/f5.aws_advanced_ha.v1.4.0rc3.tmpl" \ No newline at end of file From d077e0de2005bd164e96eef7105519bc835b658c Mon Sep 17 00:00:00 2001 From: Cody Green Date: Fri, 16 Aug 2019 11:16:46 -0500 Subject: [PATCH 11/61] added source load-key file --- deploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 7019d15..d4f048b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,5 +1,6 @@ +if [ -f "./load-keys.sh" ]; then source "./load-keys.sh"; fi if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi -if [ -z ${sshKeyName} ]; then echo "stackName is unset" && exit 1; fi +if [ -z ${sshKeyName} ]; then echo "sshKeyName is unset" && exit 1; fi if [ -z ${licenseKey1} ]; then echo "licenseKey1 is unset" && exit 1; fi if [ -z ${licenseKey2} ]; then echo "licenseKey2 is unset" && exit 1; fi if [ -z ${licenseKey3} ]; then echo "licenseKey3 is unset" && exit 1; fi From 53b4da96e0fb71f17d55388a06e8fc31c19ef3bf Mon Sep 17 00:00:00 2001 From: Cody Green Date: Fri, 16 Aug 2019 14:59:38 -0500 Subject: [PATCH 12/61] testing HA iApp config --- deprovision.sh | 1 + test.sh | 28 ++++++++-------------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/deprovision.sh b/deprovision.sh index 6914818..bb62111 100755 --- a/deprovision.sh +++ b/deprovision.sh @@ -1,3 +1,4 @@ +if [ -f "./load-keys.sh" ]; then source "./load-keys.sh"; fi if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi # delete S3 bucket content for BIG-IP clusters diff --git a/test.sh b/test.sh index f6d9a7d..8eb1e11 100755 --- a/test.sh +++ b/test.sh @@ -1,17 +1,4 @@ -# if [ -z ${BIGIP_PWD} ]; then echo "stackName is unset" && exit 1; fi -# Change admin password on BIG-IP appliances in tier-1 -# for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; -# do -# echo "set password for Tier-1 BIG-IP $ip" -# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" -# done - -# # Change admin password on BIG-IP appliances in tier-2 -# for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP2')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; -# do -# echo "set password for Tier-2 BIG-IP $ip" -# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "modify auth user admin password $BIGIP_PWD" -# done +if [ -f "./load-keys.sh" ]; then source "./load-keys.sh"; fi # Get password for BIG-IPs in tier-1 if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi @@ -19,11 +6,12 @@ S3bucket=`aws s3api list-buckets --query "[Buckets][*][?contains(Name, '$stackNa export bigip1_pwd=`aws s3 cp s3://$S3bucket/credentials/master - | jq -r .password` echo $bigip1_pwd - +# find the route table for tier1 +aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'VdssDmz1SubnetRouteTable')]" | jq -r .[][] # # install ha iApp and configure -for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; -do - echo "install HA iApp for Tier-1 BIG-IP $ip" - ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }" -done +# for ip in `aws cloudformation list-exports --query "Exports[?contains(Name, '$stackName')]|[?contains(Name, 'BIGIP1')]|[?contains(Name, 'Management')].[Value]" | jq -r .[][]`; +# do +# echo "install HA iApp for Tier-1 BIG-IP $ip" +# ssh -i $sshKey -oStrictHostKeyChecking=no admin@"$ip" "create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "rtb-06ed70ecb5a149f64" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }" +# done From d973312742dcbf0690cceacbfea60e3d336b31ee Mon Sep 17 00:00:00 2001 From: Cody Green Date: Mon, 19 Aug 2019 21:01:54 -0500 Subject: [PATCH 13/61] initial lambda commit --- lambda/.gitignore | 70 +++++++++++++++++++++++++++++++++ lambda/code/ha_iapp/README.rst | 25 ++++++++++++ lambda/code/ha_iapp/__init__.py | 0 lambda/code/ha_iapp/setup.py | 3 ++ lambda/test/.keep | 0 5 files changed, 98 insertions(+) create mode 100644 lambda/.gitignore create mode 100644 lambda/code/ha_iapp/README.rst create mode 100644 lambda/code/ha_iapp/__init__.py create mode 100644 lambda/code/ha_iapp/setup.py create mode 100644 lambda/test/.keep diff --git a/lambda/.gitignore b/lambda/.gitignore new file mode 100644 index 0000000..84da6c0 --- /dev/null +++ b/lambda/.gitignore @@ -0,0 +1,70 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ \ No newline at end of file diff --git a/lambda/code/ha_iapp/README.rst b/lambda/code/ha_iapp/README.rst new file mode 100644 index 0000000..e7bf0f0 --- /dev/null +++ b/lambda/code/ha_iapp/README.rst @@ -0,0 +1,25 @@ +ha_iapp +======= + +Lambda function to install and configure the F5 BIG-IP AWS HA iApp for a BIG-IP devices service cluster. + +Development +----------- + +1. Ensure ``Python3`` is installed as well as ``pip3`` +2. Clone repository: ``git@github.com:mikeoleary/f5-sca-securitystack.git`` +3. Install dev dependencies ``make install`` +4. Activate virtualenv: ``pipenv shell`` + +Running Tests +------------- + +Run tests locally using ``make`` if virtualenv is active: + +:: + $ make + +If virtualenv isn't active then use: + +:: + $ pipenv run make diff --git a/lambda/code/ha_iapp/__init__.py b/lambda/code/ha_iapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/code/ha_iapp/setup.py b/lambda/code/ha_iapp/setup.py new file mode 100644 index 0000000..030fa5e --- /dev/null +++ b/lambda/code/ha_iapp/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup, find_packages + +with diff --git a/lambda/test/.keep b/lambda/test/.keep new file mode 100644 index 0000000..e69de29 From 49dc8c99d7b86b1885b6ad7ff19d5f3336ba8dde Mon Sep 17 00:00:00 2001 From: Cody Green Date: Mon, 19 Aug 2019 21:51:32 -0500 Subject: [PATCH 14/61] changed lambda folder structure --- lambda/code/ha_iapp/setup.py | 3 --- lambda/{ => ha_iapp}/.gitignore | 0 lambda/ha_iapp/Makefile | 10 ++++++++++ lambda/ha_iapp/Pipfile | 11 +++++++++++ lambda/{code => }/ha_iapp/README.rst | 0 lambda/ha_iapp/setup.py | 16 ++++++++++++++++ lambda/{code => ha_iapp/src}/ha_iapp/__init__.py | 0 lambda/{test => ha_iapp/tests}/.keep | 0 8 files changed, 37 insertions(+), 3 deletions(-) delete mode 100644 lambda/code/ha_iapp/setup.py rename lambda/{ => ha_iapp}/.gitignore (100%) create mode 100644 lambda/ha_iapp/Makefile create mode 100644 lambda/ha_iapp/Pipfile rename lambda/{code => }/ha_iapp/README.rst (100%) create mode 100644 lambda/ha_iapp/setup.py rename lambda/{code => ha_iapp/src}/ha_iapp/__init__.py (100%) rename lambda/{test => ha_iapp/tests}/.keep (100%) diff --git a/lambda/code/ha_iapp/setup.py b/lambda/code/ha_iapp/setup.py deleted file mode 100644 index 030fa5e..0000000 --- a/lambda/code/ha_iapp/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup, find_packages - -with diff --git a/lambda/.gitignore b/lambda/ha_iapp/.gitignore similarity index 100% rename from lambda/.gitignore rename to lambda/ha_iapp/.gitignore diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile new file mode 100644 index 0000000..325207d --- /dev/null +++ b/lambda/ha_iapp/Makefile @@ -0,0 +1,10 @@ +.PHONY: install test + +default: test + +install: + pipenv install --dev --skip-lock + +test: + PYTHONPATH=./src pytest + diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile new file mode 100644 index 0000000..b723d01 --- /dev/null +++ b/lambda/ha_iapp/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] + +[requires] +python_version = "3.7" diff --git a/lambda/code/ha_iapp/README.rst b/lambda/ha_iapp/README.rst similarity index 100% rename from lambda/code/ha_iapp/README.rst rename to lambda/ha_iapp/README.rst diff --git a/lambda/ha_iapp/setup.py b/lambda/ha_iapp/setup.py new file mode 100644 index 0000000..da003b1 --- /dev/null +++ b/lambda/ha_iapp/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +with open ('README.rst', encoding='UTF-8') as f: + readme = f.read() + +setup( + name = 'ha_iapp', + version = '1.0.0', + description = 'Deploy and configure the F5 BIG-IP AWS HA iApp', + long_description = readme, + author = 'Cody Green', + author_email = 'c.green@f5.com', + packages = find_packages('src'), + package_dir = {'': 'src'}, + install_requires = [] +) diff --git a/lambda/code/ha_iapp/__init__.py b/lambda/ha_iapp/src/ha_iapp/__init__.py similarity index 100% rename from lambda/code/ha_iapp/__init__.py rename to lambda/ha_iapp/src/ha_iapp/__init__.py diff --git a/lambda/test/.keep b/lambda/ha_iapp/tests/.keep similarity index 100% rename from lambda/test/.keep rename to lambda/ha_iapp/tests/.keep From de9d534283a03103177ef36ffc5a63d28cc87de8 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Mon, 19 Aug 2019 23:29:05 -0500 Subject: [PATCH 15/61] get BIG-IP password from S3 bucket --- lambda/ha_iapp/Makefile | 5 +- lambda/ha_iapp/Pipfile | 3 + lambda/ha_iapp/Pipfile.lock | 107 +++++++++++++++++++++++++ lambda/ha_iapp/src/ha_iapp/password.py | 9 +++ lambda/ha_iapp/tests/test_password.py | 27 +++++++ 5 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 lambda/ha_iapp/Pipfile.lock create mode 100644 lambda/ha_iapp/src/ha_iapp/password.py create mode 100644 lambda/ha_iapp/tests/test_password.py diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile index 325207d..9e6179d 100644 --- a/lambda/ha_iapp/Makefile +++ b/lambda/ha_iapp/Makefile @@ -2,9 +2,8 @@ default: test -install: +install: pipenv install --dev --skip-lock test: - PYTHONPATH=./src pytest - + PYTHONPATH=./src pytest \ No newline at end of file diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile index b723d01..a2e1339 100644 --- a/lambda/ha_iapp/Pipfile +++ b/lambda/ha_iapp/Pipfile @@ -4,8 +4,11 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +pytest = "*" +pytest-mock = "*" [packages] +"boto3" = "*" [requires] python_version = "3.7" diff --git a/lambda/ha_iapp/Pipfile.lock b/lambda/ha_iapp/Pipfile.lock new file mode 100644 index 0000000..2401ba6 --- /dev/null +++ b/lambda/ha_iapp/Pipfile.lock @@ -0,0 +1,107 @@ +{ + "_meta": { + "hash": { + "sha256": "e8336336dbbdd3784b4f6f04c425e089571fbe19ec6852e20a6dc31ac8137585" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", + "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" + ], + "markers": "python_version < '3.8'", + "version": "==0.19" + }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, + "packaging": { + "hashes": [ + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + ], + "version": "==19.1" + }, + "pluggy": { + "hashes": [ + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + ], + "version": "==0.12.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pyparsing": { + "hashes": [ + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + ], + "version": "==2.4.2" + }, + "pytest": { + "hashes": [ + "sha256:3805d095f1ea279b9870c3eeae5dddf8a81b10952c8835cd628cf1875b0ef031", + "sha256:abc562321c2d190dd63c2faadf70b86b7af21a553b61f0df5f5e1270717dc5a3" + ], + "index": "pypi", + "version": "==5.1.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "zipp": { + "hashes": [ + "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", + "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + ], + "version": "==0.5.2" + } + } +} diff --git a/lambda/ha_iapp/src/ha_iapp/password.py b/lambda/ha_iapp/src/ha_iapp/password.py new file mode 100644 index 0000000..4e6cd7e --- /dev/null +++ b/lambda/ha_iapp/src/ha_iapp/password.py @@ -0,0 +1,9 @@ +import json + +def get(client, bucket): + """ + Get the BIG-IP admin password from the S3 bucket created by the CFT + """ + data = client.get_object(Bucket=bucket, Key="credentials/master") + json_data = data['Body'].read() + return json.loads(json_data)['password'] \ No newline at end of file diff --git a/lambda/ha_iapp/tests/test_password.py b/lambda/ha_iapp/tests/test_password.py new file mode 100644 index 0000000..5f9c010 --- /dev/null +++ b/lambda/ha_iapp/tests/test_password.py @@ -0,0 +1,27 @@ +import tempfile +import pytest +import boto3 +from botocore.stub import Stubber + +from ha_iapp import password + +@pytest.fixture +def infile(): + f = tempfile.TemporaryFile() + f.write(b'{"password": "abc123"}') + f.seek(0) + return f + +def test_get_password(mocker, infile): + """ + Read password from S3 file + """ + client = boto3.client('s3') + stubber = Stubber(client) + get_object_response = {"Body": infile} + expected_params = {'Bucket': 'bucket', 'Key': 'credentials/master'} + stubber.add_response('get_object', get_object_response, expected_params) + stubber.activate() + + pwd = password.get(client, "bucket") + assert pwd == "abc123" From 4ebec7abb42b40ffa202609675d951b8c1370e05 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 20 Aug 2019 08:43:09 -0500 Subject: [PATCH 16/61] fix format in readme --- lambda/ha_iapp/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lambda/ha_iapp/README.rst b/lambda/ha_iapp/README.rst index e7bf0f0..804e72f 100644 --- a/lambda/ha_iapp/README.rst +++ b/lambda/ha_iapp/README.rst @@ -17,9 +17,11 @@ Running Tests Run tests locally using ``make`` if virtualenv is active: :: + $ make If virtualenv isn't active then use: :: + $ pipenv run make From 9870d59c9b2f76334e63ef19491c5d4ed06ee89e Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 20 Aug 2019 08:58:39 -0500 Subject: [PATCH 17/61] add buildspec --- lambda/buildspec.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lambda/buildspec.yml diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml new file mode 100644 index 0000000..948c947 --- /dev/null +++ b/lambda/buildspec.yml @@ -0,0 +1,11 @@ +version: 0.2 + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - cd lambda/ha_iapp + - make install + - make test From 256192ec9a895d7d5d217108cbcccdef8652a04a Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 20 Aug 2019 09:02:41 -0500 Subject: [PATCH 18/61] switch make test to virtualenv --- lambda/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index 948c947..01c5580 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -8,4 +8,4 @@ phases: commands: - cd lambda/ha_iapp - make install - - make test + - pipenv run make test From 069cea8aba8e00cd0d6f82ff3edd39a072f2fcb9 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 20 Aug 2019 20:27:02 -0500 Subject: [PATCH 19/61] inital bigip commit --- lambda/ha_iapp/Pipfile | 2 + lambda/ha_iapp/src/ha_iapp/__init__.py | 70 +++++++++++++++++++++ lambda/ha_iapp/src/ha_iapp/bigip.py | 35 +++++++++++ lambda/ha_iapp/tests/test_bigip.py | 87 ++++++++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 lambda/ha_iapp/src/ha_iapp/bigip.py create mode 100644 lambda/ha_iapp/tests/test_bigip.py diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile index a2e1339..bd119ae 100644 --- a/lambda/ha_iapp/Pipfile +++ b/lambda/ha_iapp/Pipfile @@ -6,9 +6,11 @@ verify_ssl = true [dev-packages] pytest = "*" pytest-mock = "*" +mock = "*" [packages] "boto3" = "*" +"f5-sdk" = "*" [requires] python_version = "3.7" diff --git a/lambda/ha_iapp/src/ha_iapp/__init__.py b/lambda/ha_iapp/src/ha_iapp/__init__.py index e69de29..b576650 100644 --- a/lambda/ha_iapp/src/ha_iapp/__init__.py +++ b/lambda/ha_iapp/src/ha_iapp/__init__.py @@ -0,0 +1,70 @@ +import ipaddress + +def validate(*expected_args): + def decorator(f): + def wrapper(*args, **kwargs): + i = 0 + for arg in args: + if expected_args[i] == str: + validate_string(arg, 'String') + elif expected_args[i] == 'ip': + validate_ipaddress(arg, 'IP Address') + elif expected_args[i] == 'bigip': + validate_bigip(arg, 'BIG-IP') + else: + raise AttributeError(f'unsupported attribute type') + i += 1 + return f(*args, **kwargs) + return wrapper + return decorator + +def validate_wrapper(f): + def wrapper(*args, **kwargs): + kwargs['msg'] = f'a valid {args[1]} attribute is required' + f(*args, **kwargs) + return wrapper + +@validate_wrapper +def validate_string(string, name='string', msg=''): + if not string and type(string) != 'str': + # if not string: + raise AttributeError(msg) + +@validate_wrapper +def validate_ipaddress(ip, name='ip address', msg=''): + try: + ipaddress.ip_address(ip) + except ValueError: + raise AttributeError(msg) + +@validate_wrapper +def validate_bigip(bigip, name='BIG-IP', msg=''): + try: + bigip.raw() + except: + raise AttributeError(msg) + + +# def valid_bigip(f): +# def wrapper(*args, **kwargs): +# try: +# list(args)[0].raw() +# except: +# raise AttributeError(f'a valid BIG-IP ManagementRoot object is required') + +# return f(*args, **kwargs) +# return wrapper + +# def valid_ip(f): +# pass + +# def valid_string(attr_name, location, msg, empty_allowed=False): +# def decorator(f): +# def wrapper(*args, **kwargs): +# for arg in args: +# print(f'TEST: {arg}') +# if not arg: +# raise AttributeError(f'a valid {msg} is required') +# return f(*args, **kwargs) +# return wrapper +# return decorator \ No newline at end of file diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/ha_iapp/src/ha_iapp/bigip.py new file mode 100644 index 0000000..45edd5f --- /dev/null +++ b/lambda/ha_iapp/src/ha_iapp/bigip.py @@ -0,0 +1,35 @@ +from f5.bigip import ManagementRoot +from functools import wraps +from . import validate +import ipaddress + +# @valid_string("ip", 0, "IP address") +@validate({'ip': 'ip', 'user': str, 'pwd': str}) +def client(ip='', user='admin', pwd=''): + """ + create a BIG-IP client to run commands against + """ + return ManagementRoot(ip, user, pwd) + +@validate({'bigip': 'bigip', 'tmsh_cmd': str}) +def run_tmsh_cmd(bigip='', tmsh_cmd=''): + """ + Run a TMSH command against the provided bigip object + """ + return bigip.tm.sys.config.exec_cmd(tmsh_cmd) + +@validate({'bigip': 'bigip', 'bash_cmd': str}) +def run_bash_cmd(bigip='', bash_cmd=''): + """ + Run a Bash command against the provided bigip object + """ + pass + + +@validate({'bigip': 'bigip', 'iapp_name': str, 'route_table_id': str, 'interface': str}) +def cfg_ha_iapp(bigip='', iapp_name='', route_table_id='', interface=''): + """ + configure the BIG-IP AWS HA iApp + """ + tmsh_cmd = f'create /sys application service aws_HA template {iapp_name} tables add {{ subnet_routes__cidr_blocks {{ column-names {{ route_table_id dest_cidr_block }} rows {{ {{ row {{ "{route_table_id}" "0.0.0.0/0" }} }} }} }} }} variables add {{ subnet_routes__route_management {{ value yes }}, subnet_routes__interface {{ value {interface} }} }}' + return run_tmsh_cmd(bigip, tmsh_cmd) \ No newline at end of file diff --git a/lambda/ha_iapp/tests/test_bigip.py b/lambda/ha_iapp/tests/test_bigip.py new file mode 100644 index 0000000..3b6a914 --- /dev/null +++ b/lambda/ha_iapp/tests/test_bigip.py @@ -0,0 +1,87 @@ +import pytest +import mock +from ha_iapp import bigip +from f5.bigip import ManagementRoot +from icontrol.session import iControlRESTSession + + +class MockManagementRoot(ManagementRoot): + class tm(): + class sys(): + class config(): + def exec_cmd(cmd): + return '{"result": "true"}' + def raw(): + return "{}" + +@pytest.fixture +def fakebigip(): + mocker = mock.MagicMock(wraps=MockManagementRoot) + return mocker + +# ----------- BIG-IP Client --------------------- +def test_client(mocker): + mocker.patch('ha_iapp.bigip.ManagementRoot') + # create a big-ip client + client = bigip.client('192.168.100.100', 'admin', 'admin123') + bigip.ManagementRoot.assert_called_with('192.168.100.100', 'admin', 'admin123') + +def test_client_without_ip(): + with pytest.raises(AttributeError): + client = bigip.client('', 'admin', 'admin123') + +def test_client_without_pwd(): + with pytest.raises(AttributeError): + client = bigip.client('192.168.100.100', 'admin', '') + +def test_client_with_invalid_ip(): + with pytest.raises(AttributeError): + client = bigip.client('1.1.1.1.1', 'admin', 'admin123') + +# ----------- Run TMSH Command --------------------- +def test_run_tmsh_cmd_with_invalid_bigip(): + with pytest.raises(AttributeError): + bigip.run_tmsh_cmd('bigip', 'show /ltm virtual') + +def test_run_tmsh_cmd_without_tmsh_cmd(fakebigip): + with pytest.raises(AttributeError): + bigip.run_tmsh_cmd(fakebigip, '') + +def test_run_tmsh_cmd(fakebigip): + bigip.run_tmsh_cmd(fakebigip, 'show /ltm virtual') + fakebigip.tm.sys.config.exec_cmd.assert_called_with('show /ltm virtual') + +# ----------- Run Bash Command --------------------- +def test_run_bash_cmd_without_bigip(): + with pytest.raises(AttributeError): + bigip.run_bash_cmd('bigip', 'echo') + +def test_run_bash_cmd_without_bash_cmd(): + pass + +def test_run_bash_cmd(): + pass + +# ----------- Install HA iApp --------------------- + +# ----------- CFG HA iApp --------------------- +def test_cfg_ha_iapp_without_bigip(): + with pytest.raises(AttributeError): + bigip.cfg_ha_iapp('bigip', 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '/Common/internal') + +def test_cfg_ha_iapp_without_iapp_name(fakebigip): + with pytest.raises(AttributeError): + bigip.cfg_ha_iapp(fakebigip, '', 'r123456789', '/Common/internal') + +def test_cfg_ha_iapp_without_route_table_id(fakebigip): + with pytest.raises(AttributeError): + bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', '', '/Common/internal') + +def test_cfg_ha_iapp_without_interface(fakebigip): + with pytest.raises(AttributeError): + bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '') + +def test_cfg_ha_iapp(fakebigip): + bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '/Common/internal') + tmsh_cmd = 'create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "r123456789" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }' + fakebigip.tm.sys.config.exec_cmd.assert_called_with(tmsh_cmd) \ No newline at end of file From 49ae154c153637d757135f5e9b43fe9d5edf5cd5 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 20 Aug 2019 22:13:21 -0500 Subject: [PATCH 20/61] fixed validation decoration problems --- lambda/ha_iapp/src/ha_iapp/__init__.py | 46 ++++++-------------------- lambda/ha_iapp/src/ha_iapp/bigip.py | 10 +++--- 2 files changed, 14 insertions(+), 42 deletions(-) diff --git a/lambda/ha_iapp/src/ha_iapp/__init__.py b/lambda/ha_iapp/src/ha_iapp/__init__.py index b576650..b7711fb 100644 --- a/lambda/ha_iapp/src/ha_iapp/__init__.py +++ b/lambda/ha_iapp/src/ha_iapp/__init__.py @@ -3,18 +3,17 @@ def validate(*expected_args): def decorator(f): def wrapper(*args, **kwargs): - i = 0 - for arg in args: - if expected_args[i] == str: - validate_string(arg, 'String') - elif expected_args[i] == 'ip': - validate_ipaddress(arg, 'IP Address') - elif expected_args[i] == 'bigip': - validate_bigip(arg, 'BIG-IP') + zipped = zip(expected_args[0].items(), args) + for (name, arg_type), arg in zipped: + if arg_type == 'str': + validate_string(arg, name) + elif arg_type == 'ip': + validate_ipaddress(arg, name) + elif arg_type == 'bigip': + validate_bigip(arg, name) else: - raise AttributeError(f'unsupported attribute type') - i += 1 - return f(*args, **kwargs) + raise AttributeError(f'unsupported attribute type') + f(*args, **kwargs) return wrapper return decorator @@ -43,28 +42,3 @@ def validate_bigip(bigip, name='BIG-IP', msg=''): bigip.raw() except: raise AttributeError(msg) - - -# def valid_bigip(f): -# def wrapper(*args, **kwargs): -# try: -# list(args)[0].raw() -# except: -# raise AttributeError(f'a valid BIG-IP ManagementRoot object is required') - -# return f(*args, **kwargs) -# return wrapper - -# def valid_ip(f): -# pass - -# def valid_string(attr_name, location, msg, empty_allowed=False): -# def decorator(f): -# def wrapper(*args, **kwargs): -# for arg in args: -# print(f'TEST: {arg}') -# if not arg: -# raise AttributeError(f'a valid {msg} is required') -# return f(*args, **kwargs) -# return wrapper -# return decorator \ No newline at end of file diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/ha_iapp/src/ha_iapp/bigip.py index 45edd5f..ed7526f 100644 --- a/lambda/ha_iapp/src/ha_iapp/bigip.py +++ b/lambda/ha_iapp/src/ha_iapp/bigip.py @@ -1,24 +1,22 @@ from f5.bigip import ManagementRoot from functools import wraps from . import validate -import ipaddress -# @valid_string("ip", 0, "IP address") -@validate({'ip': 'ip', 'user': str, 'pwd': str}) +@validate({'ip': 'ip', 'user': 'str', 'pwd': 'str'}) def client(ip='', user='admin', pwd=''): """ create a BIG-IP client to run commands against """ return ManagementRoot(ip, user, pwd) -@validate({'bigip': 'bigip', 'tmsh_cmd': str}) +@validate({'bigip': 'bigip', 'tmsh_cmd': 'str'}) def run_tmsh_cmd(bigip='', tmsh_cmd=''): """ Run a TMSH command against the provided bigip object """ return bigip.tm.sys.config.exec_cmd(tmsh_cmd) -@validate({'bigip': 'bigip', 'bash_cmd': str}) +@validate({'bigip': 'bigip', 'bash_cmd': 'str'}) def run_bash_cmd(bigip='', bash_cmd=''): """ Run a Bash command against the provided bigip object @@ -26,7 +24,7 @@ def run_bash_cmd(bigip='', bash_cmd=''): pass -@validate({'bigip': 'bigip', 'iapp_name': str, 'route_table_id': str, 'interface': str}) +@validate({'bigip': 'bigip', 'iapp_name': 'str', 'route_table_id': 'str', 'interface': 'str'}) def cfg_ha_iapp(bigip='', iapp_name='', route_table_id='', interface=''): """ configure the BIG-IP AWS HA iApp From 25c1da66e8a6171a9b6ed2fc7bbc56ac0cf67c7a Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 08:37:53 -0500 Subject: [PATCH 21/61] fix decorator and function return issue in tests --- lambda/ha_iapp/src/ha_iapp/__init__.py | 14 +++++++++++++- lambda/ha_iapp/src/ha_iapp/bigip.py | 5 +++-- lambda/ha_iapp/src/ha_iapp/password.py | 2 ++ lambda/ha_iapp/tests/test_bigip.py | 12 +++++++----- lambda/ha_iapp/tests/test_password.py | 16 +++++++++++----- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/lambda/ha_iapp/src/ha_iapp/__init__.py b/lambda/ha_iapp/src/ha_iapp/__init__.py index b7711fb..55c88cf 100644 --- a/lambda/ha_iapp/src/ha_iapp/__init__.py +++ b/lambda/ha_iapp/src/ha_iapp/__init__.py @@ -11,9 +11,11 @@ def wrapper(*args, **kwargs): validate_ipaddress(arg, name) elif arg_type == 'bigip': validate_bigip(arg, name) + elif arg_type == 's3': + validate_s3(arg, name) else: raise AttributeError(f'unsupported attribute type') - f(*args, **kwargs) + return f(*args, **kwargs) return wrapper return decorator @@ -42,3 +44,13 @@ def validate_bigip(bigip, name='BIG-IP', msg=''): bigip.raw() except: raise AttributeError(msg) + +@validate_wrapper +def validate_s3(s3, name='S3', msg=''): + """ + There is not a boto3 S3 object, so I can't validate it. For now, make sure + it's not empty + """ + # if not s3: + # raise AttributeError(msg) + pass diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/ha_iapp/src/ha_iapp/bigip.py index ed7526f..7a5c7d5 100644 --- a/lambda/ha_iapp/src/ha_iapp/bigip.py +++ b/lambda/ha_iapp/src/ha_iapp/bigip.py @@ -1,6 +1,6 @@ from f5.bigip import ManagementRoot from functools import wraps -from . import validate +from ha_iapp import validate @validate({'ip': 'ip', 'user': 'str', 'pwd': 'str'}) def client(ip='', user='admin', pwd=''): @@ -14,7 +14,8 @@ def run_tmsh_cmd(bigip='', tmsh_cmd=''): """ Run a TMSH command against the provided bigip object """ - return bigip.tm.sys.config.exec_cmd(tmsh_cmd) + res = bigip.tm.sys.config.exec_cmd(tmsh_cmd) + return res @validate({'bigip': 'bigip', 'bash_cmd': 'str'}) def run_bash_cmd(bigip='', bash_cmd=''): diff --git a/lambda/ha_iapp/src/ha_iapp/password.py b/lambda/ha_iapp/src/ha_iapp/password.py index 4e6cd7e..5b2234f 100644 --- a/lambda/ha_iapp/src/ha_iapp/password.py +++ b/lambda/ha_iapp/src/ha_iapp/password.py @@ -1,5 +1,7 @@ import json +from ha_iapp import validate +@validate({'client': 's3', 'bucket': 'str'}) def get(client, bucket): """ Get the BIG-IP admin password from the S3 bucket created by the CFT diff --git a/lambda/ha_iapp/tests/test_bigip.py b/lambda/ha_iapp/tests/test_bigip.py index 3b6a914..516d14a 100644 --- a/lambda/ha_iapp/tests/test_bigip.py +++ b/lambda/ha_iapp/tests/test_bigip.py @@ -10,9 +10,9 @@ class tm(): class sys(): class config(): def exec_cmd(cmd): - return '{"result": "true"}' + return '{"result": "True"}' def raw(): - return "{}" + return '{"result": "true"}' @pytest.fixture def fakebigip(): @@ -48,8 +48,9 @@ def test_run_tmsh_cmd_without_tmsh_cmd(fakebigip): bigip.run_tmsh_cmd(fakebigip, '') def test_run_tmsh_cmd(fakebigip): - bigip.run_tmsh_cmd(fakebigip, 'show /ltm virtual') + res = bigip.run_tmsh_cmd(fakebigip, 'show /ltm virtual') fakebigip.tm.sys.config.exec_cmd.assert_called_with('show /ltm virtual') + assert res == '{"result": "True"}' # ----------- Run Bash Command --------------------- def test_run_bash_cmd_without_bigip(): @@ -82,6 +83,7 @@ def test_cfg_ha_iapp_without_interface(fakebigip): bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '') def test_cfg_ha_iapp(fakebigip): - bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '/Common/internal') + res = bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '/Common/internal') tmsh_cmd = 'create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "r123456789" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }' - fakebigip.tm.sys.config.exec_cmd.assert_called_with(tmsh_cmd) \ No newline at end of file + fakebigip.tm.sys.config.exec_cmd.assert_called_with(tmsh_cmd) + assert res == '{"result": "True"}' \ No newline at end of file diff --git a/lambda/ha_iapp/tests/test_password.py b/lambda/ha_iapp/tests/test_password.py index 5f9c010..f711088 100644 --- a/lambda/ha_iapp/tests/test_password.py +++ b/lambda/ha_iapp/tests/test_password.py @@ -2,7 +2,6 @@ import pytest import boto3 from botocore.stub import Stubber - from ha_iapp import password @pytest.fixture @@ -12,16 +11,23 @@ def infile(): f.seek(0) return f -def test_get_password(mocker, infile): +def test_get(infile): """ Read password from S3 file """ client = boto3.client('s3') stubber = Stubber(client) - get_object_response = {"Body": infile} + get_object_response = {'Body': infile} expected_params = {'Bucket': 'bucket', 'Key': 'credentials/master'} stubber.add_response('get_object', get_object_response, expected_params) stubber.activate() + pwd = password.get(client, 'bucket') + assert pwd == 'abc123' - pwd = password.get(client, "bucket") - assert pwd == "abc123" +def test_get_without_client(): + + with pytest.raises(AttributeError): + password.get('', 'bucket') + +def test_get_without_buciket(): + pass From 2345242b315849a42cc68579a8fd7c69d0a8f867 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 13:09:47 -0500 Subject: [PATCH 22/61] fixed implementation and test for run_bash_cmd --- lambda/ha_iapp/src/ha_iapp/__init__.py | 2 +- lambda/ha_iapp/src/ha_iapp/bigip.py | 6 +++--- lambda/ha_iapp/tests/test_bigip.py | 19 ++++++++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lambda/ha_iapp/src/ha_iapp/__init__.py b/lambda/ha_iapp/src/ha_iapp/__init__.py index 55c88cf..dc08328 100644 --- a/lambda/ha_iapp/src/ha_iapp/__init__.py +++ b/lambda/ha_iapp/src/ha_iapp/__init__.py @@ -41,7 +41,7 @@ def validate_ipaddress(ip, name='ip address', msg=''): @validate_wrapper def validate_bigip(bigip, name='BIG-IP', msg=''): try: - bigip.raw() + bigip.raw except: raise AttributeError(msg) diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/ha_iapp/src/ha_iapp/bigip.py index 7a5c7d5..839b907 100644 --- a/lambda/ha_iapp/src/ha_iapp/bigip.py +++ b/lambda/ha_iapp/src/ha_iapp/bigip.py @@ -14,15 +14,15 @@ def run_tmsh_cmd(bigip='', tmsh_cmd=''): """ Run a TMSH command against the provided bigip object """ - res = bigip.tm.sys.config.exec_cmd(tmsh_cmd) - return res + return bigip.tm.sys.config.exec_cmd(tmsh_cmd) @validate({'bigip': 'bigip', 'bash_cmd': 'str'}) def run_bash_cmd(bigip='', bash_cmd=''): """ Run a Bash command against the provided bigip object """ - pass + res = bigip.tm.util.bash.exec_cmd('run', utilCmdArgs=f'-c "{bash_cmd}"') + return res.commandResult @validate({'bigip': 'bigip', 'iapp_name': 'str', 'route_table_id': 'str', 'interface': 'str'}) diff --git a/lambda/ha_iapp/tests/test_bigip.py b/lambda/ha_iapp/tests/test_bigip.py index 516d14a..0eeae4b 100644 --- a/lambda/ha_iapp/tests/test_bigip.py +++ b/lambda/ha_iapp/tests/test_bigip.py @@ -11,8 +11,15 @@ class sys(): class config(): def exec_cmd(cmd): return '{"result": "True"}' + class util(): + class bash(): + def exec_cmd(action, utilCmdArgs): + class output: + commandResult = '{"result": True")' + result = output() + return result def raw(): - return '{"result": "true"}' + return '{"result": "True"}' @pytest.fixture def fakebigip(): @@ -57,11 +64,13 @@ def test_run_bash_cmd_without_bigip(): with pytest.raises(AttributeError): bigip.run_bash_cmd('bigip', 'echo') -def test_run_bash_cmd_without_bash_cmd(): - pass +def test_run_bash_cmd_without_bash_cmd(fakebigip): + with pytest.raises(AttributeError): + bigip.run_bash_cmd(fakebigip, '') -def test_run_bash_cmd(): - pass +def test_run_bash_cmd(fakebigip): + res = bigip.run_bash_cmd(fakebigip, 'netstat -rn') + fakebigip.tm.util.bash.exec_cmd.assert_called_with('run', utilCmdArgs='-c "netstat -rn"') # ----------- Install HA iApp --------------------- From f8d649d91ed1b93c7811ec56a0579bf4dea2a48d Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 14:26:19 -0500 Subject: [PATCH 23/61] added install_iapp and fixed cfg_ha_iapp --- lambda/ha_iapp/src/ha_iapp/bigip.py | 23 ++++++++++++++++++++--- lambda/ha_iapp/tests/test_bigip.py | 15 +++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/ha_iapp/src/ha_iapp/bigip.py index 839b907..4ad5057 100644 --- a/lambda/ha_iapp/src/ha_iapp/bigip.py +++ b/lambda/ha_iapp/src/ha_iapp/bigip.py @@ -22,13 +22,30 @@ def run_bash_cmd(bigip='', bash_cmd=''): Run a Bash command against the provided bigip object """ res = bigip.tm.util.bash.exec_cmd('run', utilCmdArgs=f'-c "{bash_cmd}"') - return res.commandResult + + # determine if there was output from the command + if 'commandResult' in res.__dict__: + return res.commandResult + else: + return True + +@validate({'bigip': 'bigip', 'url': 'str'}) +def install_iapp(bigip='', url=''): + """ + download and install an iApp + """ + bash_cmd = f'curl {url} -o /config/cloud/aws/{url.split("/")[-1]}' + download_res = run_bash_cmd(bigip, bash_cmd) + + load_cmd = f'tmsh load sys application template /config/cloud/aws/{url.split("/")[-1]}' + load_res = run_bash_cmd(bigip, bash_cmd) + return load_res @validate({'bigip': 'bigip', 'iapp_name': 'str', 'route_table_id': 'str', 'interface': 'str'}) def cfg_ha_iapp(bigip='', iapp_name='', route_table_id='', interface=''): """ configure the BIG-IP AWS HA iApp """ - tmsh_cmd = f'create /sys application service aws_HA template {iapp_name} tables add {{ subnet_routes__cidr_blocks {{ column-names {{ route_table_id dest_cidr_block }} rows {{ {{ row {{ "{route_table_id}" "0.0.0.0/0" }} }} }} }} }} variables add {{ subnet_routes__route_management {{ value yes }}, subnet_routes__interface {{ value {interface} }} }}' - return run_tmsh_cmd(bigip, tmsh_cmd) \ No newline at end of file + cfg_cmd = f'tmsh create /sys application service aws_HA template {iapp_name} tables add {{ subnet_routes__cidr_blocks {{ column-names {{ route_table_id dest_cidr_block }} rows {{ {{ row {{ "{route_table_id}" "0.0.0.0/0" }} }} }} }} }} variables add {{ subnet_routes__route_management {{ value yes }}, subnet_routes__interface {{ value {interface} }} }}' + return run_bash_cmd(bigip, cfg_cmd) \ No newline at end of file diff --git a/lambda/ha_iapp/tests/test_bigip.py b/lambda/ha_iapp/tests/test_bigip.py index 0eeae4b..6cd3c37 100644 --- a/lambda/ha_iapp/tests/test_bigip.py +++ b/lambda/ha_iapp/tests/test_bigip.py @@ -73,6 +73,11 @@ def test_run_bash_cmd(fakebigip): fakebigip.tm.util.bash.exec_cmd.assert_called_with('run', utilCmdArgs='-c "netstat -rn"') # ----------- Install HA iApp --------------------- +def test_install_iapp(fakebigip): + url = 'https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl' + res = bigip.install_iapp(fakebigip, url) + assert res == True + # ----------- CFG HA iApp --------------------- def test_cfg_ha_iapp_without_bigip(): @@ -92,7 +97,9 @@ def test_cfg_ha_iapp_without_interface(fakebigip): bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '') def test_cfg_ha_iapp(fakebigip): - res = bigip.cfg_ha_iapp(fakebigip, 'f5.aws_advanced_ha.v1.4.0rc3', 'r123456789', '/Common/internal') - tmsh_cmd = 'create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "r123456789" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }' - fakebigip.tm.sys.config.exec_cmd.assert_called_with(tmsh_cmd) - assert res == '{"result": "True"}' \ No newline at end of file + iapp_name = 'f5.aws_advanced_ha.v1.4.0rc3' + route_table_id = 'r123456789' + interface = '/Common/internal' + res = bigip.cfg_ha_iapp(fakebigip, iapp_name, route_table_id, interface) + fakebigip.tm.util.bash.exec_cmd.assert_called_with('run', utilCmdArgs='-c "tmsh create /sys application service aws_HA template f5.aws_advanced_ha.v1.4.0rc3 tables add { subnet_routes__cidr_blocks { column-names { route_table_id dest_cidr_block } rows { { row { "r123456789" "0.0.0.0/0" } } } } } variables add { subnet_routes__route_management { value yes }, subnet_routes__interface { value /Common/internal } }"') + assert res == True \ No newline at end of file From e3047fad358287cdd324cdedf0a022095e1ed287 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 14:51:15 -0500 Subject: [PATCH 24/61] move code for reuse between lambda functions --- lambda/buildspec.yml | 2 +- lambda/{ha_iapp => f5_sca_libs}/Makefile | 0 lambda/{ha_iapp => f5_sca_libs}/Pipfile | 0 lambda/{ha_iapp => f5_sca_libs}/Pipfile.lock | 0 lambda/{ha_iapp => f5_sca_libs}/README.rst | 2 +- lambda/{ha_iapp => f5_sca_libs}/setup.py | 2 +- .../src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/__init__.py | 0 .../src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/bigip.py | 2 +- .../src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/password.py | 2 +- lambda/{ha_iapp => f5_sca_libs}/tests/test_bigip.py | 4 ++-- lambda/{ha_iapp => f5_sca_libs}/tests/test_password.py | 2 +- lambda/ha_iapp/tests/.keep | 0 12 files changed, 8 insertions(+), 8 deletions(-) rename lambda/{ha_iapp => f5_sca_libs}/Makefile (100%) rename lambda/{ha_iapp => f5_sca_libs}/Pipfile (100%) rename lambda/{ha_iapp => f5_sca_libs}/Pipfile.lock (100%) rename lambda/{ha_iapp => f5_sca_libs}/README.rst (97%) rename lambda/{ha_iapp => f5_sca_libs}/setup.py (93%) rename lambda/{ha_iapp/src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/__init__.py (100%) rename lambda/{ha_iapp/src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/bigip.py (98%) rename lambda/{ha_iapp/src/ha_iapp => f5_sca_libs/src/f5_sca_libs}/password.py (90%) rename lambda/{ha_iapp => f5_sca_libs}/tests/test_bigip.py (98%) rename lambda/{ha_iapp => f5_sca_libs}/tests/test_password.py (95%) delete mode 100644 lambda/ha_iapp/tests/.keep diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index 01c5580..82d6186 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -6,6 +6,6 @@ phases: python: 3.7 build: commands: - - cd lambda/ha_iapp + - cd lambda/f5_sca_libs - make install - pipenv run make test diff --git a/lambda/ha_iapp/Makefile b/lambda/f5_sca_libs/Makefile similarity index 100% rename from lambda/ha_iapp/Makefile rename to lambda/f5_sca_libs/Makefile diff --git a/lambda/ha_iapp/Pipfile b/lambda/f5_sca_libs/Pipfile similarity index 100% rename from lambda/ha_iapp/Pipfile rename to lambda/f5_sca_libs/Pipfile diff --git a/lambda/ha_iapp/Pipfile.lock b/lambda/f5_sca_libs/Pipfile.lock similarity index 100% rename from lambda/ha_iapp/Pipfile.lock rename to lambda/f5_sca_libs/Pipfile.lock diff --git a/lambda/ha_iapp/README.rst b/lambda/f5_sca_libs/README.rst similarity index 97% rename from lambda/ha_iapp/README.rst rename to lambda/f5_sca_libs/README.rst index 804e72f..d189dfb 100644 --- a/lambda/ha_iapp/README.rst +++ b/lambda/f5_sca_libs/README.rst @@ -1,4 +1,4 @@ -ha_iapp +f5_sca_libs ======= Lambda function to install and configure the F5 BIG-IP AWS HA iApp for a BIG-IP devices service cluster. diff --git a/lambda/ha_iapp/setup.py b/lambda/f5_sca_libs/setup.py similarity index 93% rename from lambda/ha_iapp/setup.py rename to lambda/f5_sca_libs/setup.py index da003b1..fa02082 100644 --- a/lambda/ha_iapp/setup.py +++ b/lambda/f5_sca_libs/setup.py @@ -4,7 +4,7 @@ readme = f.read() setup( - name = 'ha_iapp', + name = 'f5_sca_libs', version = '1.0.0', description = 'Deploy and configure the F5 BIG-IP AWS HA iApp', long_description = readme, diff --git a/lambda/ha_iapp/src/ha_iapp/__init__.py b/lambda/f5_sca_libs/src/f5_sca_libs/__init__.py similarity index 100% rename from lambda/ha_iapp/src/ha_iapp/__init__.py rename to lambda/f5_sca_libs/src/f5_sca_libs/__init__.py diff --git a/lambda/ha_iapp/src/ha_iapp/bigip.py b/lambda/f5_sca_libs/src/f5_sca_libs/bigip.py similarity index 98% rename from lambda/ha_iapp/src/ha_iapp/bigip.py rename to lambda/f5_sca_libs/src/f5_sca_libs/bigip.py index 4ad5057..87464c7 100644 --- a/lambda/ha_iapp/src/ha_iapp/bigip.py +++ b/lambda/f5_sca_libs/src/f5_sca_libs/bigip.py @@ -1,6 +1,6 @@ from f5.bigip import ManagementRoot from functools import wraps -from ha_iapp import validate +from f5_sca_libs import validate @validate({'ip': 'ip', 'user': 'str', 'pwd': 'str'}) def client(ip='', user='admin', pwd=''): diff --git a/lambda/ha_iapp/src/ha_iapp/password.py b/lambda/f5_sca_libs/src/f5_sca_libs/password.py similarity index 90% rename from lambda/ha_iapp/src/ha_iapp/password.py rename to lambda/f5_sca_libs/src/f5_sca_libs/password.py index 5b2234f..2330067 100644 --- a/lambda/ha_iapp/src/ha_iapp/password.py +++ b/lambda/f5_sca_libs/src/f5_sca_libs/password.py @@ -1,5 +1,5 @@ import json -from ha_iapp import validate +from f5_sca_libs import validate @validate({'client': 's3', 'bucket': 'str'}) def get(client, bucket): diff --git a/lambda/ha_iapp/tests/test_bigip.py b/lambda/f5_sca_libs/tests/test_bigip.py similarity index 98% rename from lambda/ha_iapp/tests/test_bigip.py rename to lambda/f5_sca_libs/tests/test_bigip.py index 6cd3c37..369b05c 100644 --- a/lambda/ha_iapp/tests/test_bigip.py +++ b/lambda/f5_sca_libs/tests/test_bigip.py @@ -1,6 +1,6 @@ import pytest import mock -from ha_iapp import bigip +from f5_sca_libs import bigip from f5.bigip import ManagementRoot from icontrol.session import iControlRESTSession @@ -28,7 +28,7 @@ def fakebigip(): # ----------- BIG-IP Client --------------------- def test_client(mocker): - mocker.patch('ha_iapp.bigip.ManagementRoot') + mocker.patch('f5_sca_libs.bigip.ManagementRoot') # create a big-ip client client = bigip.client('192.168.100.100', 'admin', 'admin123') bigip.ManagementRoot.assert_called_with('192.168.100.100', 'admin', 'admin123') diff --git a/lambda/ha_iapp/tests/test_password.py b/lambda/f5_sca_libs/tests/test_password.py similarity index 95% rename from lambda/ha_iapp/tests/test_password.py rename to lambda/f5_sca_libs/tests/test_password.py index f711088..757fa62 100644 --- a/lambda/ha_iapp/tests/test_password.py +++ b/lambda/f5_sca_libs/tests/test_password.py @@ -2,7 +2,7 @@ import pytest import boto3 from botocore.stub import Stubber -from ha_iapp import password +from f5_sca_libs import password @pytest.fixture def infile(): diff --git a/lambda/ha_iapp/tests/.keep b/lambda/ha_iapp/tests/.keep deleted file mode 100644 index e69de29..0000000 From 83e108dc41578c34f42b24e8d13a91319ecdd77c Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 15:09:59 -0500 Subject: [PATCH 25/61] create python wheel --- .gitignore | 5 +++++ lambda/f5_sca_libs/MANIFEST.in | 2 ++ lambda/f5_sca_libs/Makefile | 7 +++++-- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 lambda/f5_sca_libs/MANIFEST.in diff --git a/.gitignore b/.gitignore index 9b63e7e..f110caf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .DS_Store load-keys.sh + +# lambda +lambda/f5_sca_libs/build +lambda/f5_sca_libs/dist +lambda/f5_sca_libs/src/*.egg-info \ No newline at end of file diff --git a/lambda/f5_sca_libs/MANIFEST.in b/lambda/f5_sca_libs/MANIFEST.in new file mode 100644 index 0000000..343583c --- /dev/null +++ b/lambda/f5_sca_libs/MANIFEST.in @@ -0,0 +1,2 @@ +include README.rst +resursive-include tests *.py \ No newline at end of file diff --git a/lambda/f5_sca_libs/Makefile b/lambda/f5_sca_libs/Makefile index 9e6179d..e665f7f 100644 --- a/lambda/f5_sca_libs/Makefile +++ b/lambda/f5_sca_libs/Makefile @@ -1,4 +1,4 @@ -.PHONY: install test +.PHONY: install test wheel default: test @@ -6,4 +6,7 @@ install: pipenv install --dev --skip-lock test: - PYTHONPATH=./src pytest \ No newline at end of file + PYTHONPATH=./src pytest + +wheel: + pipenv run python setup.py bdist_wheel \ No newline at end of file From 2f460b2bbeeb0b249f5f905ade3787bb1789233c Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 15:33:43 -0500 Subject: [PATCH 26/61] adding make wheel to pipeline --- lambda/buildspec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index 82d6186..5e7b9f8 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -9,3 +9,4 @@ phases: - cd lambda/f5_sca_libs - make install - pipenv run make test + - make wheel From 192f27cd4fea0af5ef9d97889de04323c676ddc2 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 15:39:12 -0500 Subject: [PATCH 27/61] added artifact --- lambda/buildspec.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index 5e7b9f8..773d8c8 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -10,3 +10,6 @@ phases: - make install - pipenv run make test - make wheel +artifacts: + files: + - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl \ No newline at end of file From 80c32da3daae0a7606f99afcc5cf86f40e427fa0 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 15:51:48 -0500 Subject: [PATCH 28/61] change artifact directory --- lambda/buildspec.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index 773d8c8..dbf70d6 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -12,4 +12,6 @@ phases: - make wheel artifacts: files: - - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl \ No newline at end of file + - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl + discard-paths: yes + base-directory: 'lambda' \ No newline at end of file From a7554685d16e8b9d2888b155592f18d971e80aa9 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 15:54:17 -0500 Subject: [PATCH 29/61] remove base-directory --- lambda/buildspec.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index dbf70d6..b23cccd 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -14,4 +14,3 @@ artifacts: files: - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl discard-paths: yes - base-directory: 'lambda' \ No newline at end of file From a979676315e5632fb27567489eea606c9d612367 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 19:52:57 -0500 Subject: [PATCH 30/61] added ha_iapp lambda function --- .gitignore | 3 ++- lambda/buildspec.yml | 4 ++++ lambda/f5_sca_libs/README.rst | 2 +- lambda/ha_iapp/Makefile | 13 +++++++++++++ lambda/ha_iapp/Pipfile | 12 ++++++++++++ lambda/ha_iapp/Pipfile.lock | 28 ++++++++++++++++++++++++++++ lambda/ha_iapp/README.rst | 27 +++++++++++++++++++++++++++ lambda/ha_iapp/index.py | 27 +++++++++++++++++++++++++++ 8 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 lambda/ha_iapp/Makefile create mode 100644 lambda/ha_iapp/Pipfile create mode 100644 lambda/ha_iapp/Pipfile.lock create mode 100644 lambda/ha_iapp/README.rst create mode 100644 lambda/ha_iapp/index.py diff --git a/.gitignore b/.gitignore index f110caf..cbca6ba 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ load-keys.sh # lambda lambda/f5_sca_libs/build lambda/f5_sca_libs/dist -lambda/f5_sca_libs/src/*.egg-info \ No newline at end of file +lambda/f5_sca_libs/src/*.egg-info +lambda/ha_iapp/ha_iapp.zip diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index b23cccd..e59c2bc 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -10,7 +10,11 @@ phases: - make install - pipenv run make test - make wheel + - cd ../ha_iapp + - make install + - make zip artifacts: files: - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl + - lambda/ha_iapp/ha_iapp.zip discard-paths: yes diff --git a/lambda/f5_sca_libs/README.rst b/lambda/f5_sca_libs/README.rst index d189dfb..7a0cdb7 100644 --- a/lambda/f5_sca_libs/README.rst +++ b/lambda/f5_sca_libs/README.rst @@ -1,7 +1,7 @@ f5_sca_libs ======= -Lambda function to install and configure the F5 BIG-IP AWS HA iApp for a BIG-IP devices service cluster. +Python library to help deploy the F5 Secure Cloud Arhictecture in AWS Development ----------- diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile new file mode 100644 index 0000000..d67d991 --- /dev/null +++ b/lambda/ha_iapp/Makefile @@ -0,0 +1,13 @@ +.PHONY: install test wheel + +default: test + +install: + pipenv install --dev --skip-lock + +zip: + pipenv lock -r > requirements.txt + pipenv run pip install -r requirements.txt --no-deps -t ./ + zip -FSr ha_iapp.zip ./* + pipenv run pip uninstall -r requirements.txt -y + rm -rf requirements.txt f5_sca_libs* \ No newline at end of file diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile new file mode 100644 index 0000000..ece1eab --- /dev/null +++ b/lambda/ha_iapp/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +f5-sca-libs = {path = "./../f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl"} + +[requires] +python_version = "3.7" diff --git a/lambda/ha_iapp/Pipfile.lock b/lambda/ha_iapp/Pipfile.lock new file mode 100644 index 0000000..9c27888 --- /dev/null +++ b/lambda/ha_iapp/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "620213b39ff2b0e010c0297372fb5f98d2581e45291451df04be4701be41892f" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "f5-sca-libs": { + "hashes": [ + "sha256:2c51b3a7b8566b4babe232028806d216a2e93a7be7721819924dc43a60fa4b7a" + ], + "path": "./../f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl", + "version": "==1.0.0" + } + }, + "develop": {} +} diff --git a/lambda/ha_iapp/README.rst b/lambda/ha_iapp/README.rst new file mode 100644 index 0000000..804e72f --- /dev/null +++ b/lambda/ha_iapp/README.rst @@ -0,0 +1,27 @@ +ha_iapp +======= + +Lambda function to install and configure the F5 BIG-IP AWS HA iApp for a BIG-IP devices service cluster. + +Development +----------- + +1. Ensure ``Python3`` is installed as well as ``pip3`` +2. Clone repository: ``git@github.com:mikeoleary/f5-sca-securitystack.git`` +3. Install dev dependencies ``make install`` +4. Activate virtualenv: ``pipenv shell`` + +Running Tests +------------- + +Run tests locally using ``make`` if virtualenv is active: + +:: + + $ make + +If virtualenv isn't active then use: + +:: + + $ pipenv run make diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py new file mode 100644 index 0000000..e277085 --- /dev/null +++ b/lambda/ha_iapp/index.py @@ -0,0 +1,27 @@ +from f5_sca_libs import bigip +from f5_sca_libs import password +import boto3 +import json + +def lambda_handler(event, context): + print("Received event: " + json.dumps(event, indent=2)) + + # obtain required variables + mgmt_ip = event['Records'][0]['bigip']['mgmt_ip'] + iapp_url = event['Records'][0]['bigip']['iapp_url'] + route_table_id = event['Records'][0]['bigip']['route_table_id'] + interface = event['Records'][0]['bigip']['interface'] + s3_bucket = event['Records'][0]['bigip']['s3_bucket'] + + if not mgmt_ip or not iapp_url or not route_table_id or not interface or not s3_bucket: + raise Exception(f'ERROR: missing required event attributes {json.dumps(event, indent=2)} ') + + # obtain BIG-IP Password + s3_client = boto3.client('s3') + password = password.get(s3_client, s3_bucket) + + # deploy and configure ha_iapp + client = bigip.client(mgmt_ip, 'admin', password) + bigip.install_iapp(client, iapp_url) + iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') + bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) \ No newline at end of file From 15290ec2b6211f7eb4e36c9ae29c8a74d3e3871f Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 21:43:10 -0500 Subject: [PATCH 31/61] fixed lambda packaging issues --- .gitignore | 3 ++ lambda/ha_iapp/Makefile | 6 ++-- lambda/ha_iapp/Pipfile | 1 + lambda/ha_iapp/Pipfile.lock | 57 ++++++++++++++++++++++++++++++++++++- lambda/ha_iapp/index.py | 36 ++++++++++++----------- 5 files changed, 84 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index cbca6ba..5b198e0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ lambda/f5_sca_libs/build lambda/f5_sca_libs/dist lambda/f5_sca_libs/src/*.egg-info lambda/ha_iapp/ha_iapp.zip +lambda/ha_iapp/bin +lambda/ha_iapp/certifi* +lambda/ha_iapp/ \ No newline at end of file diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile index d67d991..cbe918f 100644 --- a/lambda/ha_iapp/Makefile +++ b/lambda/ha_iapp/Makefile @@ -7,7 +7,9 @@ install: zip: pipenv lock -r > requirements.txt - pipenv run pip install -r requirements.txt --no-deps -t ./ + pipenv run pip install -r requirements.txt -t ./ + chmod 644 $$(find ./ -type f) + chmod 755 $$(find ./ -type d) zip -FSr ha_iapp.zip ./* pipenv run pip uninstall -r requirements.txt -y - rm -rf requirements.txt f5_sca_libs* \ No newline at end of file + rm -rf requirements.txt f5_sca_libs* bin ch* dev* certifi* f5* ic* id* re* setup* six* url* \ No newline at end of file diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile index ece1eab..b45490d 100644 --- a/lambda/ha_iapp/Pipfile +++ b/lambda/ha_iapp/Pipfile @@ -6,6 +6,7 @@ verify_ssl = true [dev-packages] [packages] +f5-sdk = "*" f5-sca-libs = {path = "./../f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl"} [requires] diff --git a/lambda/ha_iapp/Pipfile.lock b/lambda/ha_iapp/Pipfile.lock index 9c27888..ef31303 100644 --- a/lambda/ha_iapp/Pipfile.lock +++ b/lambda/ha_iapp/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "620213b39ff2b0e010c0297372fb5f98d2581e45291451df04be4701be41892f" + "sha256": "d3aad29b8075a6664f5344bd305a74421cf905f93f23099dc97b26935414e4f5" }, "pipfile-spec": 6, "requires": { @@ -16,12 +16,67 @@ ] }, "default": { + "certifi": { + "hashes": [ + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + ], + "version": "==2019.6.16" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "f5-icontrol-rest": { + "hashes": [ + "sha256:49fffd999fb4971d6754beb0e066051175db9d9baeb8a76fca6c801dacc89359" + ], + "version": "==1.3.13" + }, "f5-sca-libs": { "hashes": [ "sha256:2c51b3a7b8566b4babe232028806d216a2e93a7be7721819924dc43a60fa4b7a" ], "path": "./../f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl", "version": "==1.0.0" + }, + "f5-sdk": { + "hashes": [ + "sha256:22890c8fd98232c14c54560ee02a59501da2df670e6a6852e6eda6364363963a" + ], + "index": "pypi", + "version": "==3.0.21" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + ], + "version": "==1.25.3" } }, "develop": {} diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py index e277085..ef5adde 100644 --- a/lambda/ha_iapp/index.py +++ b/lambda/ha_iapp/index.py @@ -2,26 +2,30 @@ from f5_sca_libs import password import boto3 import json +import logging def lambda_handler(event, context): print("Received event: " + json.dumps(event, indent=2)) - # obtain required variables - mgmt_ip = event['Records'][0]['bigip']['mgmt_ip'] - iapp_url = event['Records'][0]['bigip']['iapp_url'] - route_table_id = event['Records'][0]['bigip']['route_table_id'] - interface = event['Records'][0]['bigip']['interface'] - s3_bucket = event['Records'][0]['bigip']['s3_bucket'] + if event['RequestType'] == 'Create': + # obtain required variables + mgmt_ip = event['ResourceProperties'].get('mgmt_ip') + iapp_url = event['ResourceProperties'].get('iapp_url') + route_table_id = event['ResourceProperties'].get('route_table_id') + interface = event['ResourceProperties'].get('interface') + s3_bucket = event['ResourceProperties'].get('s3_bucket') - if not mgmt_ip or not iapp_url or not route_table_id or not interface or not s3_bucket: - raise Exception(f'ERROR: missing required event attributes {json.dumps(event, indent=2)} ') + if not mgmt_ip or not iapp_url or not route_table_id or not interface or not s3_bucket: + raise Exception(f'ERROR: missing required event attributes {json.dumps(event, indent=2)} ') - # obtain BIG-IP Password - s3_client = boto3.client('s3') - password = password.get(s3_client, s3_bucket) + # obtain BIG-IP Password + s3_client = boto3.client('s3') + pwd = password.get(s3_client, s3_bucket) + if not pwd: + raise Exception('ERROR: unable to obtain BIG-IP password') - # deploy and configure ha_iapp - client = bigip.client(mgmt_ip, 'admin', password) - bigip.install_iapp(client, iapp_url) - iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') - bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) \ No newline at end of file + # deploy and configure ha_iapp + client = bigip.client(mgmt_ip, 'admin', pwd) + bigip.install_iapp(client, iapp_url) + iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') + bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) \ No newline at end of file From 80e0f46b6b9e877d0611c95e275f57ea63714b8f Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 21:43:49 -0500 Subject: [PATCH 32/61] remove .gitignore restrictions --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 5b198e0..ef40574 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,4 @@ load-keys.sh lambda/f5_sca_libs/build lambda/f5_sca_libs/dist lambda/f5_sca_libs/src/*.egg-info -lambda/ha_iapp/ha_iapp.zip -lambda/ha_iapp/bin -lambda/ha_iapp/certifi* -lambda/ha_iapp/ \ No newline at end of file +lambda/ha_iapp/ha_iapp.zip \ No newline at end of file From 7c2521772c7fbf5f2a0fe0686760ffb440f30050 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 21 Aug 2019 22:37:23 -0500 Subject: [PATCH 33/61] created CFT to deploy the ha iapp --- BIG-IP/ha_iapp.json | 178 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 BIG-IP/ha_iapp.json diff --git a/BIG-IP/ha_iapp.json b/BIG-IP/ha_iapp.json new file mode 100644 index 0000000..77c377f --- /dev/null +++ b/BIG-IP/ha_iapp.json @@ -0,0 +1,178 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Parameters": { + "pVpcStackName": { + "Type": "String", + "Default": "SCCA-Demo", + "Description": "VPC Stack used for referencing exported resources" + }, + "pBigIPTier1Mgmt": { + "Type": "String", + "Default": "", + "Description": "BIGIP Management IP address for Tier1" + }, + "pBigIPTier2Mgmt": { + "Type": "String", + "Default": "", + "Description": "BIGIP Management IP address for Tier2" + }, + "pIAppUrl": { + "Type": "String", + "Default": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl", + "Description": "URL to download BIG-IP AWS HA iApp" + }, + "pBigIPTier1RouteTableId": { + "Type": "String", + "Default": "", + "Description": "Route Table ID for Tier1 HA iApp" + }, + "pBigIPTier2RouteTableId": { + "Type": "String", + "Default": "", + "Description": "Route Table ID for Tier2 HA iApp" + }, + "pBigIPTier1Interface": { + "Type": "String", + "Default": "/Common/internal", + "Description": "Interface for Tier1 HA iApp" + }, + "pBigIPTier2Interface": { + "Type": "String", + "Default": "/Common/external", + "Description": "Interface for Tier2 HA iApp" + }, + "pBigIPTier1S3Bucket": { + "Type": "String", + "Default": "", + "Description": "S3 Bucket where BIG-IP password is stored" + }, + "pBigIPTier2S3Bucket": { + "Type": "String", + "Default": "", + "Description": "S3 Bucket where BIG-IP password is stored" + }, + "pQuickstartS3BucketName": { + "Type": "String", + "Default": "f5-sca-securitystack", + "Description": "S3 bucket name for the Quick Start assets. Quick Start bucket name can include numbers, lowercase letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-)." + }, + "pQuickstartS3KeyPrefix": { + "Type": "String", + "Default": "master", + "Description": "Quick Start key prefix can include numbers, lowercase letters, uppercase letters, hyphens (-), and forward slash (/)." + } + }, + "Resources": { + "IamRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + }, + "Action": [ + "sts:AssumeRole" + ] + } + ] + }, + "Path": "/", + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/AmazonVPCFullAccess", + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" + ] + } + }, + "CustomDeployHAiAppTier1": { + "Type": "Custom::DeployHAiApp", + "Version": "1.0", + "Properties": { + "ServiceToken": { + "Fn:GetAtt": [ + "LambdaHAiApp", + "Arn" + ] + }, + "mgmt_ip": { + "Ref": "pBigIPTier1Mgmt" + }, + "iapp_url": { + "Ref": "pIAppUrl" + }, + "route_table_id": { + "Ref": "pBigIPTier1RouteTableId" + }, + "interface": { + "Ref": "pBigIPTier1Interface" + }, + "s3_bucket": { + "Ref": "pBigIPTier1S3Bucket" + } + } + }, + "CustomDeployHAiAppTier2": { + "Type": "Custom::DeployHAiApp", + "Version": "1.0", + "Properties": { + "ServiceToken": { + "Fn:GetAtt": [ + "LambdaHAiApp", + "Arn" + ] + }, + "mgmt_ip": { + "Ref": "pBigIPTier2Mgmt" + }, + "iapp_url": { + "Ref": "pIAppUrl" + }, + "route_table_id": { + "Ref": "pBigIPTier2RouteTableId" + }, + "interface": { + "Ref": "pBigIPTier2Interface" + }, + "s3_bucket": { + "Ref": "pBigIPTier2S3Bucket" + } + } + }, + "LambdaHAiApp": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Handler": "index.lambda_handler", + "Role": { + "Fn::GetAtt": [ + "IamRole", + "Arn" + ] + }, + "Runtime": "python3.6", + "Timeout": 30, + "Code": { + "S3Bucket": { + "Ref": "pQuickstartS3BucketName" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Ref": "pQuickstartS3KeyPrefix" + }, + "/ha_iapp.zip" + ] + ] + } + } + } + } + } +} \ No newline at end of file From f25c57f89e20e72d5d8d50c21da63eb6686ba45a Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 15:04:13 -0500 Subject: [PATCH 34/61] cft factory creation --- .gitignore | 5 +- cft-factory/Makefile | 10 +++ cft-factory/Pipfile | 11 +++ cft-factory/ha_iapp/gen_cft.py | 104 +++++++++++++++++++++++++++ cft-factory/ha_iapp/ha_iapp.json | 117 +++++++++++++++++++++++++++++++ lambda/buildspec.yml | 7 ++ 6 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 cft-factory/Makefile create mode 100644 cft-factory/Pipfile create mode 100644 cft-factory/ha_iapp/gen_cft.py create mode 100644 cft-factory/ha_iapp/ha_iapp.json diff --git a/.gitignore b/.gitignore index ef40574..43097ce 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ load-keys.sh lambda/f5_sca_libs/build lambda/f5_sca_libs/dist lambda/f5_sca_libs/src/*.egg-info -lambda/ha_iapp/ha_iapp.zip \ No newline at end of file +lambda/ha_iapp/ha_iapp.zip + +# CFT Factory +cft-factory/ha_iapp.json diff --git a/cft-factory/Makefile b/cft-factory/Makefile new file mode 100644 index 0000000..d6da15b --- /dev/null +++ b/cft-factory/Makefile @@ -0,0 +1,10 @@ +.PHONY: install cft + +default: cft + +install: + cd ha_iapp + pipenv install --dev --skip-lock + +cft: + pipenv run python ./ha_iapp/gen_cft.py \ No newline at end of file diff --git a/cft-factory/Pipfile b/cft-factory/Pipfile new file mode 100644 index 0000000..b723d01 --- /dev/null +++ b/cft-factory/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] + +[requires] +python_version = "3.7" diff --git a/cft-factory/ha_iapp/gen_cft.py b/cft-factory/ha_iapp/gen_cft.py new file mode 100644 index 0000000..3bf88ce --- /dev/null +++ b/cft-factory/ha_iapp/gen_cft.py @@ -0,0 +1,104 @@ +from troposphere import GetAtt, Join, Ref, Template, Parameter +from troposphere.iam import Role +from troposphere.cloudformation import AWSCustomObject, CustomResource +from troposphere.awslambda import Code, Function + +from awacs.aws import Allow, Statement, Principal, PolicyDocument +from awacs.sts import AssumeRole + +template = Template() + +# add parameters +piAppUrl = template.add_parameter(Parameter( + 'piAppUrl', + Description='URL to download the BIG-IP AWS HA iApp', + Default='https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl', + Type='String' +)) + +pBigIPMgmt = template.add_parameter(Parameter( + 'pBigIPMgmt', + Description='Active BIG-IP Management IP address', + Type='String' +)) + +pBigIPRouteTableId = template.add_parameter(Parameter( + 'pBigIPRouteTableId', + Description='BIG-IP route table id for HA', + Type='String' +)) + +pBigIPInterface = template.add_parameter(Parameter( + 'pBigIPInterface', + Description='Tier 1 BIG-IP interface for HA', + Type='String' +)) + + +pBigIPS3Bucket = template.add_parameter(Parameter( + 'pBigIPS3Bucket', + Description='BIG-IP S3 bucket where password is stored', + Type='String' +)) + +pLambdaS3BucketName = template.add_parameter(Parameter( + 'pLambdaS3BucketName', + Description='S3 bucket where lambda code is stored', + Type='String' +)) + +pLambdaS3KeyPrefix = template.add_parameter(Parameter( + 'pLambdaS3KeyPrefix', + Description='S3 Key Prefix where lambda code is stored', + Type='String' +)) + +# add IAM Role +iamRole = template.add_resource(Role( + 'LambdaRole', + AssumeRolePolicyDocument=PolicyDocument( + Statement=[ + Statement( + Effect=Allow, + Action=[AssumeRole], + Principal=Principal('Service', ['lambda.amazonaws.com']) + ) + ] + ), + Path='/', + ManagedPolicyArns = [ + 'arn:aws:iam::aws:policy/AmazonVPCFullAccess', + 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', + 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess' + ] +)) + +custom = template.add_resource(CustomResource( + 'CustomLambdaExec', + ServiceToken=GetAtt('HAiApp', 'Arn'), + mgmt_ip=Ref('pBigIPMgmt'), + iapp_url=Ref('piAppUrl'), + route_table_id=Ref('pBigIPRouteTableId'), + interface=Ref('pBigIPInterface'), + s3_bucket=Ref('pBigIPS3Bucket') +)) + +ha_lambda = template.add_resource(Function( + 'HAiApp', + Handler= 'index.lambda_handler', + Role= GetAtt('LambdaRole', 'Arn'), + Code= Code( + S3Bucket=Ref('pLambdaS3BucketName'), + S3Key=Join("", [ + Ref('pLambdaS3KeyPrefix'), + "/ha_iapp.zip" + ]) + ), + Runtime='python3.6', + Timeout=30 +)) + +# print(template.to_yaml()) +f = open('ha_iapp.json', "w+") +f.write(template.to_json()) +f.close() \ No newline at end of file diff --git a/cft-factory/ha_iapp/ha_iapp.json b/cft-factory/ha_iapp/ha_iapp.json new file mode 100644 index 0000000..1ee7e3f --- /dev/null +++ b/cft-factory/ha_iapp/ha_iapp.json @@ -0,0 +1,117 @@ +{ + "Parameters": { + "pBigIPInterface": { + "Description": "Tier 1 BIG-IP interface for HA", + "Type": "String" + }, + "pBigIPMgmt": { + "Description": "Active BIG-IP Management IP address", + "Type": "String" + }, + "pBigIPRouteTableId": { + "Description": "BIG-IP route table id for HA", + "Type": "String" + }, + "pBigIPS3Bucket": { + "Description": "BIG-IP S3 bucket where password is stored", + "Type": "String" + }, + "pLambdaS3BucketName": { + "Description": "S3 bucket where lambda code is stored", + "Type": "String" + }, + "pLambdaS3KeyPrefix": { + "Description": "S3 Key Prefix where lambda code is stored", + "Type": "String" + }, + "piAppUrl": { + "Default": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl", + "Description": "URL to download the BIG-IP AWS HA iApp", + "Type": "String" + } + }, + "Resources": { + "CustomLambdaExec": { + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "HAiApp", + "Arn" + ] + }, + "iapp_url": { + "Ref": "piAppUrl" + }, + "interface": { + "Ref": "pBigIPInterface" + }, + "mgmt_ip": { + "Ref": "pBigIPMgmt" + }, + "route_table_id": { + "Ref": "pBigIPRouteTableId" + }, + "s3_bucket": { + "Ref": "pBigIPS3Bucket" + } + }, + "Type": "AWS::CloudFormation::CustomResource" + }, + "HAiApp": { + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "pLambdaS3BucketName" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Ref": "pLambdaS3KeyPrefix" + }, + "/ha_iapp.zip" + ] + ] + } + }, + "Handler": "index.lambda_handler", + "Role": { + "Fn::GetAtt": [ + "LambdaRole", + "Arn" + ] + }, + "Runtime": "python3.6", + "Timeout": 30 + }, + "Type": "AWS::Lambda::Function" + }, + "LambdaRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + } + ] + }, + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/AmazonVPCFullAccess", + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" + ], + "Path": "/" + }, + "Type": "AWS::IAM::Role" + } + } +} \ No newline at end of file diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index e59c2bc..f4a7b2b 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -6,15 +6,22 @@ phases: python: 3.7 build: commands: + # test and build f5_sca_libs - cd lambda/f5_sca_libs - make install - pipenv run make test - make wheel + # test and zip ha_iapp lambda - cd ../ha_iapp - make install - make zip + # make ha_iapp CFT + - cd ../cft-factory + - make install + - make cft artifacts: files: - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl - lambda/ha_iapp/ha_iapp.zip + - cft-factory/ha_iapp.json discard-paths: yes From d6e2cc2ae05821f3c65dfcfb652e8eb822816f0a Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 15:09:22 -0500 Subject: [PATCH 35/61] fixed path issue --- lambda/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index f4a7b2b..cd537dc 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -16,7 +16,7 @@ phases: - make install - make zip # make ha_iapp CFT - - cd ../cft-factory + - cd ../../cft-factory - make install - make cft artifacts: From 22a426550f9c38b22a71bb3bb13478f7bcc1baee Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 16:06:42 -0500 Subject: [PATCH 36/61] fixed path issue --- lambda/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index cd537dc..c8ff04b 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -16,7 +16,7 @@ phases: - make install - make zip # make ha_iapp CFT - - cd ../../cft-factory + - cd ../../cft-factory/ha_iapp - make install - make cft artifacts: From f25e5a1396876ff8110506ae03b623a758157a89 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 16:44:31 -0500 Subject: [PATCH 37/61] fix pipenv install issue --- cft-factory/Makefile | 5 +- cft-factory/Pipfile | 2 + .../gen_cft.py => gen_ha_iapp_cft.py} | 0 cft-factory/ha_iapp/ha_iapp.json | 117 ------------------ 4 files changed, 4 insertions(+), 120 deletions(-) rename cft-factory/{ha_iapp/gen_cft.py => gen_ha_iapp_cft.py} (100%) delete mode 100644 cft-factory/ha_iapp/ha_iapp.json diff --git a/cft-factory/Makefile b/cft-factory/Makefile index d6da15b..d6d7be9 100644 --- a/cft-factory/Makefile +++ b/cft-factory/Makefile @@ -3,8 +3,7 @@ default: cft install: - cd ha_iapp - pipenv install --dev --skip-lock + pipenv install --skip-lock cft: - pipenv run python ./ha_iapp/gen_cft.py \ No newline at end of file + pipenv run python gen_ha_iapp_cft.py \ No newline at end of file diff --git a/cft-factory/Pipfile b/cft-factory/Pipfile index b723d01..0804a46 100644 --- a/cft-factory/Pipfile +++ b/cft-factory/Pipfile @@ -6,6 +6,8 @@ verify_ssl = true [dev-packages] [packages] +troposphere = "*" +awacs = "*" [requires] python_version = "3.7" diff --git a/cft-factory/ha_iapp/gen_cft.py b/cft-factory/gen_ha_iapp_cft.py similarity index 100% rename from cft-factory/ha_iapp/gen_cft.py rename to cft-factory/gen_ha_iapp_cft.py diff --git a/cft-factory/ha_iapp/ha_iapp.json b/cft-factory/ha_iapp/ha_iapp.json deleted file mode 100644 index 1ee7e3f..0000000 --- a/cft-factory/ha_iapp/ha_iapp.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "Parameters": { - "pBigIPInterface": { - "Description": "Tier 1 BIG-IP interface for HA", - "Type": "String" - }, - "pBigIPMgmt": { - "Description": "Active BIG-IP Management IP address", - "Type": "String" - }, - "pBigIPRouteTableId": { - "Description": "BIG-IP route table id for HA", - "Type": "String" - }, - "pBigIPS3Bucket": { - "Description": "BIG-IP S3 bucket where password is stored", - "Type": "String" - }, - "pLambdaS3BucketName": { - "Description": "S3 bucket where lambda code is stored", - "Type": "String" - }, - "pLambdaS3KeyPrefix": { - "Description": "S3 Key Prefix where lambda code is stored", - "Type": "String" - }, - "piAppUrl": { - "Default": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl", - "Description": "URL to download the BIG-IP AWS HA iApp", - "Type": "String" - } - }, - "Resources": { - "CustomLambdaExec": { - "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "HAiApp", - "Arn" - ] - }, - "iapp_url": { - "Ref": "piAppUrl" - }, - "interface": { - "Ref": "pBigIPInterface" - }, - "mgmt_ip": { - "Ref": "pBigIPMgmt" - }, - "route_table_id": { - "Ref": "pBigIPRouteTableId" - }, - "s3_bucket": { - "Ref": "pBigIPS3Bucket" - } - }, - "Type": "AWS::CloudFormation::CustomResource" - }, - "HAiApp": { - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "pLambdaS3BucketName" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Ref": "pLambdaS3KeyPrefix" - }, - "/ha_iapp.zip" - ] - ] - } - }, - "Handler": "index.lambda_handler", - "Role": { - "Fn::GetAtt": [ - "LambdaRole", - "Arn" - ] - }, - "Runtime": "python3.6", - "Timeout": 30 - }, - "Type": "AWS::Lambda::Function" - }, - "LambdaRole": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } - } - ] - }, - "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/AmazonVPCFullAccess", - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" - ], - "Path": "/" - }, - "Type": "AWS::IAM::Role" - } - } -} \ No newline at end of file From 617f4f9d8cc1283561f0e15cbeabb86ead0fce7c Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 16:49:16 -0500 Subject: [PATCH 38/61] fix buildspec path issue --- lambda/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index c8ff04b..cd537dc 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -16,7 +16,7 @@ phases: - make install - make zip # make ha_iapp CFT - - cd ../../cft-factory/ha_iapp + - cd ../../cft-factory - make install - make cft artifacts: From 96120b721baeda2d5df9190a6807af4437e509f5 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 17:06:56 -0500 Subject: [PATCH 39/61] added CFT resposes to lambda --- lambda/ha_iapp/Pipfile | 1 + lambda/ha_iapp/Pipfile.lock | 41 ++++++++++++++++++++++++++++++++++++- lambda/ha_iapp/index.py | 7 ++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lambda/ha_iapp/Pipfile b/lambda/ha_iapp/Pipfile index b45490d..4c6e9d1 100644 --- a/lambda/ha_iapp/Pipfile +++ b/lambda/ha_iapp/Pipfile @@ -8,6 +8,7 @@ verify_ssl = true [packages] f5-sdk = "*" f5-sca-libs = {path = "./../f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl"} +cfnresponse = "*" [requires] python_version = "3.7" diff --git a/lambda/ha_iapp/Pipfile.lock b/lambda/ha_iapp/Pipfile.lock index ef31303..7a521ba 100644 --- a/lambda/ha_iapp/Pipfile.lock +++ b/lambda/ha_iapp/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d3aad29b8075a6664f5344bd305a74421cf905f93f23099dc97b26935414e4f5" + "sha256": "891262227706e06af8a1d438d78618d7c0ea622e486a1aeaa2ec36dc5b727daa" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,13 @@ ] }, "default": { + "botocore": { + "hashes": [ + "sha256:1c357c8081f67797596eb28295cac38aa835cdb4ef6c69ede57e203359659d9c", + "sha256:2b6e366cac12f7146542ac802ea62a6b46dfc92e9ee3eb75ba450949d5e10f71" + ], + "version": "==1.12.214" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", @@ -23,6 +30,14 @@ ], "version": "==2019.6.16" }, + "cfnresponse": { + "hashes": [ + "sha256:5feb11df87317f91f668a2d4815f3a920f19cffb89215b072cfd378f966ca92b", + "sha256:8691302ac5d7511a979b07e6c1413ee8de815c9575c8f8a4c187cdc21ca2deca" + ], + "index": "pypi", + "version": "==1.0.1" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -30,6 +45,14 @@ ], "version": "==3.0.4" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, "f5-icontrol-rest": { "hashes": [ "sha256:49fffd999fb4971d6754beb0e066051175db9d9baeb8a76fca6c801dacc89359" @@ -57,6 +80,21 @@ ], "version": "==2.8" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, "requests": { "hashes": [ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", @@ -76,6 +114,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" } }, diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py index ef5adde..e9f093e 100644 --- a/lambda/ha_iapp/index.py +++ b/lambda/ha_iapp/index.py @@ -28,4 +28,9 @@ def lambda_handler(event, context): client = bigip.client(mgmt_ip, 'admin', pwd) bigip.install_iapp(client, iapp_url) iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') - bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) \ No newline at end of file + bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) + + cfnresponse.send(event, context, cfnresponse.SUCCESS, True) + + if event['RequestType'] == 'Delete': + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) \ No newline at end of file From 2b18e780c07a15ca40126d7cb3762550de444e9d Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 19:21:38 -0500 Subject: [PATCH 40/61] seperated lambda install and added ha_iapp to parent cft --- aws-quickstart-scca-main-same-net.json | 316 ++++++++++++++++++------- cft-factory/gen_ha_iapp_cft.py | 50 +--- cft-factory/gen_install_lambda_cft.py | 61 +++++ lambda/buildspec.yml | 1 + 4 files changed, 291 insertions(+), 137 deletions(-) create mode 100644 cft-factory/gen_install_lambda_cft.py diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index d7a1465..7dbcb12 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -51,22 +51,22 @@ "MinLength": 9 }, "pAppVpcCidr": { - "Type": "String", - "Default": "10.1.0.0/16", - "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", - "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x.", - "Description": "CIDR Block for the app VPC", - "MaxLength": 18, - "MinLength": 9 + "Type": "String", + "Default": "10.1.0.0/16", + "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", + "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x.", + "Description": "CIDR Block for the app VPC", + "MaxLength": 18, + "MinLength": 9 }, "pFargateVpcCidr": { - "Type": "String", - "Default": "10.2.0.0/16", - "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", - "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x.", - "Description": "CIDR Block for the app VPC", - "MaxLength": 18, - "MinLength": 9 + "Type": "String", + "Default": "10.2.0.0/16", + "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", + "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x.", + "Description": "CIDR Block for the app VPC", + "MaxLength": 18, + "MinLength": 9 }, "licenseKey1": { "AllowedPattern": "([\\x41-\\x5A][\\x41-\\x5A|\\x30-\\x39]{4})\\-([\\x41-\\x5A|\\x30-\\x39]{5})\\-([\\x41-\\x5A|\\x30-\\x39]{5})\\-([\\x41-\\x5A|\\x30-\\x39]{5})\\-([\\x41-\\x5A|\\x30-\\x39]{7})", @@ -149,7 +149,7 @@ }, "VdssStack": { "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", + "DependsOn": "TransitGatewayStack", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -190,92 +190,92 @@ } }, "AppVpcStack": { - "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", - "Properties": { - "TemplateURL": { - "Fn::Sub": [ - "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/VPC/aws-scca-app-vpc.json", - { - "S3Region": { - "Fn::If": [ - "GovCloud", - "s3-us-gov-west-1", - "s3" - ] - } - } - ] - }, - "Parameters": { - "pTransitGatewayStackName": { - "Fn::GetAtt": [ - "TransitGatewayStack", - "Outputs.StackName" - ] - }, - "pAppVpcCidr": { - "Ref": "pAppVpcCidr" - } - }, - "Tags": [ + "Type": "AWS::CloudFormation::Stack", + "DependsOn": "TransitGatewayStack", + "Properties": { + "TemplateURL": { + "Fn::Sub": [ + "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/VPC/aws-scca-app-vpc.json", { - "Key": "Project", - "Value": { - "Ref": "pProject" + "S3Region": { + "Fn::If": [ + "GovCloud", + "s3-us-gov-west-1", + "s3" + ] } } ] }, - "Metadata": { - "aws:cdk:path": "sccaMainStack/AppVPCStack" - } - }, - "FargateVpcStack": { - "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", - "Properties": { - "TemplateURL": { - "Fn::Sub": [ - "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/VPC/aws-scca-fargate-vpc.json", - { - "S3Region": { - "Fn::If": [ - "GovCloud", - "s3-us-gov-west-1", - "s3" - ] - } - } + "Parameters": { + "pTransitGatewayStackName": { + "Fn::GetAtt": [ + "TransitGatewayStack", + "Outputs.StackName" ] }, - "Parameters": { - "pTransitGatewayStackName": { - "Fn::GetAtt": [ - "TransitGatewayStack", - "Outputs.StackName" - ] - }, - "pFargateVpcCidr": { - "Ref": "pFargateVpcCidr" + "pAppVpcCidr": { + "Ref": "pAppVpcCidr" + } + }, + "Tags": [ + { + "Key": "Project", + "Value": { + "Ref": "pProject" } - }, - "Tags": [ + } + ] + }, + "Metadata": { + "aws:cdk:path": "sccaMainStack/AppVPCStack" + } + }, + "FargateVpcStack": { + "Type": "AWS::CloudFormation::Stack", + "DependsOn": "TransitGatewayStack", + "Properties": { + "TemplateURL": { + "Fn::Sub": [ + "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/VPC/aws-scca-fargate-vpc.json", { - "Key": "Project", - "Value": { - "Ref": "pProject" + "S3Region": { + "Fn::If": [ + "GovCloud", + "s3-us-gov-west-1", + "s3" + ] } } ] }, - "Metadata": { - "aws:cdk:path": "sccaMainStack/FargateVPCStack" - } + "Parameters": { + "pTransitGatewayStackName": { + "Fn::GetAtt": [ + "TransitGatewayStack", + "Outputs.StackName" + ] + }, + "pFargateVpcCidr": { + "Ref": "pFargateVpcCidr" + } + }, + "Tags": [ + { + "Key": "Project", + "Value": { + "Ref": "pProject" + } + } + ] + }, + "Metadata": { + "aws:cdk:path": "sccaMainStack/FargateVPCStack" + } }, "JumpHostStack": { "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", + "DependsOn": "TransitGatewayStack", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -318,7 +318,7 @@ }, "F5BIGIP1": { "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", + "DependsOn": "TransitGatewayStack", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -392,7 +392,7 @@ }, "F5BIGIP2": { "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", + "DependsOn": "TransitGatewayStack", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -466,7 +466,7 @@ }, "IPSStack": { "Type": "AWS::CloudFormation::Stack", - "DependsOn" : "TransitGatewayStack", + "DependsOn": "TransitGatewayStack", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -562,6 +562,146 @@ "Metadata": { "aws:cdk:path": "sccaMainStack/RouteUpdatesStack" } + }, + "InstallLambda": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Sub": [ + "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/install_lambda.json", + { + "S3Region": { + "Fn::If": [ + "GovCloud", + "s3-us-gov-west-1", + "s3" + ] + } + } + ] + }, + "Parameters": { + "pLambdaS3BucketName": { + "Fn::Ref": "pQuickstartS3BucketName" + }, + "pLambdaS3KeyPrefix": { + "Fn::Ref": "pQuickstartS3KeyPrefix" + } + }, + "Tags": [ + { + "Key": "Project", + "Value": { + "Ref": "pProject" + } + } + ] + }, + "Metadata": { + "aws:cdk:path": "sccaMainStack/InstallLambdaStack" + } + }, + "HAiAppTier1": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Sub": [ + "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/ha_iapp.json", + { + "S3Region": { + "Fn::If": [ + "GovCloud", + "s3-us-gov-west-1", + "s3" + ] + } + } + ] + }, + "Parameters": { + "pBigIPMgmt": { + "Fn::GetAtt": [ + "F5BIGIP1", + "Outputs.Bigip1ManagementEipAddress" + ] + }, + "pBigIPRouteTableId": { + "Fn::GetAtt": [ + "VdssStack", + "VdssDmz1SubnetRouteTable" + ] + }, + "pBigIPInterface": "/Common/internal", + "pBigIPS3Bucket": { + "Fn::GetAtt": [ + "F5BIGIP1", + "Outputs.S3BucketName" + ] + } + }, + "Tags": [ + { + "Key": "Project", + "Value": { + "Ref": "pProject" + } + } + ] + }, + "Metadata": { + "aws:cdk:path": "sccaMainStack/InstallLambdaStack" + } + }, + "HAiAppTier2": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Sub": [ + "https://${pQuickstartS3BucketName}.${S3Region}.amazonaws.com/${pQuickstartS3KeyPrefix}/ha_iapp.json", + { + "S3Region": { + "Fn::If": [ + "GovCloud", + "s3-us-gov-west-1", + "s3" + ] + } + } + ] + }, + "Parameters": { + "pBigIPMgmt": { + "Fn::GetAtt": [ + "F5BIGIP2", + "Outputs.Bigip1ManagementEipAddress" + ] + }, + "pBigIPRouteTableId": { + "Fn::GetAtt": [ + "VdssStack", + "VdssDmz2SubnetRouteTable" + ] + }, + "pBigIPInterface": "/Common/internal", + "pBigIPS3Bucket": { + "Fn::GetAtt": [ + "F5BIGIP2", + "Outputs.S3BucketName" + ] + } + }, + "Tags": [ + { + "Key": "Project", + "Value": { + "Ref": "pProject" + } + } + ] + }, + "Metadata": { + "aws:cdk:path": "sccaMainStack/InstallLambdaStack" + } } } } \ No newline at end of file diff --git a/cft-factory/gen_ha_iapp_cft.py b/cft-factory/gen_ha_iapp_cft.py index 3bf88ce..0998f97 100644 --- a/cft-factory/gen_ha_iapp_cft.py +++ b/cft-factory/gen_ha_iapp_cft.py @@ -1,5 +1,4 @@ from troposphere import GetAtt, Join, Ref, Template, Parameter -from troposphere.iam import Role from troposphere.cloudformation import AWSCustomObject, CustomResource from troposphere.awslambda import Code, Function @@ -41,38 +40,6 @@ Type='String' )) -pLambdaS3BucketName = template.add_parameter(Parameter( - 'pLambdaS3BucketName', - Description='S3 bucket where lambda code is stored', - Type='String' -)) - -pLambdaS3KeyPrefix = template.add_parameter(Parameter( - 'pLambdaS3KeyPrefix', - Description='S3 Key Prefix where lambda code is stored', - Type='String' -)) - -# add IAM Role -iamRole = template.add_resource(Role( - 'LambdaRole', - AssumeRolePolicyDocument=PolicyDocument( - Statement=[ - Statement( - Effect=Allow, - Action=[AssumeRole], - Principal=Principal('Service', ['lambda.amazonaws.com']) - ) - ] - ), - Path='/', - ManagedPolicyArns = [ - 'arn:aws:iam::aws:policy/AmazonVPCFullAccess', - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', - 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess' - ] -)) - custom = template.add_resource(CustomResource( 'CustomLambdaExec', ServiceToken=GetAtt('HAiApp', 'Arn'), @@ -83,22 +50,7 @@ s3_bucket=Ref('pBigIPS3Bucket') )) -ha_lambda = template.add_resource(Function( - 'HAiApp', - Handler= 'index.lambda_handler', - Role= GetAtt('LambdaRole', 'Arn'), - Code= Code( - S3Bucket=Ref('pLambdaS3BucketName'), - S3Key=Join("", [ - Ref('pLambdaS3KeyPrefix'), - "/ha_iapp.zip" - ]) - ), - Runtime='python3.6', - Timeout=30 -)) - -# print(template.to_yaml()) +# print(template.to_json()) f = open('ha_iapp.json', "w+") f.write(template.to_json()) f.close() \ No newline at end of file diff --git a/cft-factory/gen_install_lambda_cft.py b/cft-factory/gen_install_lambda_cft.py new file mode 100644 index 0000000..9382b3d --- /dev/null +++ b/cft-factory/gen_install_lambda_cft.py @@ -0,0 +1,61 @@ +from troposphere import GetAtt, Join, Ref, Template +from troposphere.iam import Role +from troposphere.awslambda import Code, Function + +from awacs.aws import Allow, Statement, Principal, PolicyDocument +from awacs.sts import AssumeRole + +template = Template() + +# add parameters +pLambdaS3BucketName = template.add_parameter(Parameter( + 'pLambdaS3BucketName', + Description='S3 bucket where lambda code is stored', + Type='String' +)) + +pLambdaS3KeyPrefix = template.add_parameter(Parameter( + 'pLambdaS3KeyPrefix', + Description='S3 Key Prefix where lambda code is stored', + Type='String' +)) + +# add IAM Role +iamRole = template.add_resource(Role( + 'LambdaRole', + AssumeRolePolicyDocument=PolicyDocument( + Statement=[ + Statement( + Effect=Allow, + Action=[AssumeRole], + Principal=Principal('Service', ['lambda.amazonaws.com']) + ) + ] + ), + Path='/', + ManagedPolicyArns = [ + 'arn:aws:iam::aws:policy/AmazonVPCFullAccess', + 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', + 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess' + ] +)) + +ha_lambda = template.add_resource(Function( + 'HAiApp', + Handler= 'index.lambda_handler', + Role= GetAtt('LambdaRole', 'Arn'), + Code= Code( + S3Bucket=Ref('pLambdaS3BucketName'), + S3Key=Join("", [ + Ref('pLambdaS3KeyPrefix'), + "/ha_iapp.zip" + ]) + ), + Runtime='python3.6', + Timeout=30 +)) + +# print(template.to_yaml()) +f = open('install_lambda.json', "w+") +f.write(template.to_json()) +f.close() \ No newline at end of file diff --git a/lambda/buildspec.yml b/lambda/buildspec.yml index cd537dc..b709957 100644 --- a/lambda/buildspec.yml +++ b/lambda/buildspec.yml @@ -24,4 +24,5 @@ artifacts: - lambda/f5_sca_libs/dist/f5_sca_libs-1.0.0-py3-none-any.whl - lambda/ha_iapp/ha_iapp.zip - cft-factory/ha_iapp.json + - cft-factory/install_lambda.json discard-paths: yes From 8e2836a0cef8c73e0c8b4ae11f53886c2bdbddff Mon Sep 17 00:00:00 2001 From: Cody Green Date: Thu, 22 Aug 2019 19:27:17 -0500 Subject: [PATCH 41/61] add DependsOn to BIG-IP stacks --- aws-quickstart-scca-main-same-net.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index 7dbcb12..bb504a2 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -603,6 +603,7 @@ }, "HAiAppTier1": { "Type": "AWS::CloudFormation::Stack", + "DependsOn": "F5BIGIP1", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -654,6 +655,7 @@ }, "HAiAppTier2": { "Type": "AWS::CloudFormation::Stack", + "DependsOn": "F5BIGIP2", "Properties": { "TemplateURL": { "Fn::Sub": [ From b6042d8ed483c4ee8c22116d92716d23e1462ed7 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 09:18:26 -0500 Subject: [PATCH 42/61] fix S3 error: The specified key does not exist for lambda install --- aws-quickstart-scca-main-same-net.json | 12 +- cft-factory/Pipfile | 1 + cft-factory/Pipfile.lock | 200 +++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 cft-factory/Pipfile.lock diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index bb504a2..d3fe60d 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -580,14 +580,6 @@ } ] }, - "Parameters": { - "pLambdaS3BucketName": { - "Fn::Ref": "pQuickstartS3BucketName" - }, - "pLambdaS3KeyPrefix": { - "Fn::Ref": "pQuickstartS3KeyPrefix" - } - }, "Tags": [ { "Key": "Project", @@ -629,7 +621,7 @@ "pBigIPRouteTableId": { "Fn::GetAtt": [ "VdssStack", - "VdssDmz1SubnetRouteTable" + "Outputs.VdssDmz1SubnetRouteTable" ] }, "pBigIPInterface": "/Common/internal", @@ -681,7 +673,7 @@ "pBigIPRouteTableId": { "Fn::GetAtt": [ "VdssStack", - "VdssDmz2SubnetRouteTable" + "Outputs.VdssDmz2SubnetRouteTable" ] }, "pBigIPInterface": "/Common/internal", diff --git a/cft-factory/Pipfile b/cft-factory/Pipfile index 0804a46..e50b8af 100644 --- a/cft-factory/Pipfile +++ b/cft-factory/Pipfile @@ -4,6 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +cfn-lint = "*" [packages] troposphere = "*" diff --git a/cft-factory/Pipfile.lock b/cft-factory/Pipfile.lock new file mode 100644 index 0000000..2e69e85 --- /dev/null +++ b/cft-factory/Pipfile.lock @@ -0,0 +1,200 @@ +{ + "_meta": { + "hash": { + "sha256": "cc667df7db032f401a2937ee0e952ab3dad6432222c8790c84656312ebda7d8c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "awacs": { + "hashes": [ + "sha256:52681404f2731e93473a436b43375a30562f17a3bfc0064f986c2fb2aa73102a" + ], + "index": "pypi", + "version": "==0.9.6" + }, + "cfn-flip": { + "hashes": [ + "sha256:3660e488411ba0ded256992292b16a830811626293418cdb5db7290e8f40b6ac", + "sha256:43e73032afeb5f87b78958e13159e676e99db698033b8de2f0433657388c46aa" + ], + "version": "==1.2.1" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "troposphere": { + "hashes": [ + "sha256:2a0ab9f02fd28ca3c38ba363c1774ca672afab24558d6024a8145a107980b747" + ], + "index": "pypi", + "version": "==2.5.1" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "aws-sam-translator": { + "hashes": [ + "sha256:b38639c2b7527f664da4e18e09d67a691e8e3877ae0ca1196a0d129a020a3676" + ], + "version": "==1.13.2" + }, + "boto3": { + "hashes": [ + "sha256:3701eb13dbee1de3ecc6ed3c725929993933aa2ffdbb6fe8a4985d5c94d6a67a", + "sha256:d32a19f1e373cfe7f1fbd22137489f69e08e61a904a09bd7235fbadcfc647789" + ], + "version": "==1.9.216" + }, + "botocore": { + "hashes": [ + "sha256:63ca29cf8a47b190886923c789a3f2b2a7e1665b5f6e22a97fed057af2e5f1af", + "sha256:bebb869754459035fe45eac1f8a93a97dd8698522bc7f7ebde791e32311b035a" + ], + "version": "==1.12.216" + }, + "cfn-lint": { + "hashes": [ + "sha256:2384c67de57bb7fdaecc657223786bd3ea62ba1f63e8baf7f87488d8ccbb82aa", + "sha256:cd2be1ba2c1f25a500e02570b975e68bdeba665f87e64ab3a0eac30c74da049c" + ], + "index": "pypi", + "version": "==0.23.4" + }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, + "jsonpatch": { + "hashes": [ + "sha256:83f29a2978c13da29bfdf89da9d65542d62576479caf215df19632d7dc04c6e6", + "sha256:cbb72f8bf35260628aea6b508a107245f757d1ec839a19c34349985e2c05645a" + ], + "version": "==1.24" + }, + "jsonpointer": { + "hashes": [ + "sha256:c192ba86648e05fdae4f08a17ec25180a9aef5008d973407b581798a83975362", + "sha256:ff379fa021d1b81ab539f5ec467c7745beb1a5671463f9dcc2b2d458bd361c1e" + ], + "version": "==2.0" + }, + "jsonschema": { + "hashes": [ + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" + ], + "version": "==3.0.2" + }, + "pyrsistent": { + "hashes": [ + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" + ], + "version": "==0.15.4" + }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" + }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + ], + "markers": "python_version >= '3.4'", + "version": "==1.25.3" + } + } +} From fa72961d3f274bbd5e41bddc2b76552625cded5a Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 09:28:54 -0500 Subject: [PATCH 43/61] fix lambda install cft creation errors --- .gitignore | 3 ++- cft-factory/Makefile | 3 ++- cft-factory/gen_install_lambda_cft.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 43097ce..adce0eb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ lambda/f5_sca_libs/dist lambda/f5_sca_libs/src/*.egg-info lambda/ha_iapp/ha_iapp.zip -# CFT Factory +# CFT Factory - these files are generated cft-factory/ha_iapp.json +cft-factory/install_lambda.json diff --git a/cft-factory/Makefile b/cft-factory/Makefile index d6d7be9..33a943b 100644 --- a/cft-factory/Makefile +++ b/cft-factory/Makefile @@ -6,4 +6,5 @@ install: pipenv install --skip-lock cft: - pipenv run python gen_ha_iapp_cft.py \ No newline at end of file + pipenv run python gen_ha_iapp_cft.py + pipenv run python gen_install_lambda_cft.py \ No newline at end of file diff --git a/cft-factory/gen_install_lambda_cft.py b/cft-factory/gen_install_lambda_cft.py index 9382b3d..39cad02 100644 --- a/cft-factory/gen_install_lambda_cft.py +++ b/cft-factory/gen_install_lambda_cft.py @@ -1,4 +1,4 @@ -from troposphere import GetAtt, Join, Ref, Template +from troposphere import GetAtt, Join, Ref, Template, Parameter from troposphere.iam import Role from troposphere.awslambda import Code, Function From e80a93e102305176e71d110c16ecf5a4df4a823b Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 09:35:27 -0500 Subject: [PATCH 44/61] fix unassigned parameters for lambda install --- aws-quickstart-scca-main-same-net.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index d3fe60d..f14d32b 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -580,6 +580,10 @@ } ] }, + "Parameters": { + "pLambdaS3BucketName": "${pQuickstartS3BucketName}", + "pLambdaS3KeyPrefix": "${pQuickstartS3KeyPrefix}" + }, "Tags": [ { "Key": "Project", From b6c42e316419132b87c7b764de539ef91903593f Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 10:49:38 -0500 Subject: [PATCH 45/61] fix install lambda parameter issue, address ha_iapp ref error --- aws-quickstart-scca-main-same-net.json | 8 ++++++-- cft-factory/gen_ha_iapp_cft.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index f14d32b..df45995 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -581,8 +581,12 @@ ] }, "Parameters": { - "pLambdaS3BucketName": "${pQuickstartS3BucketName}", - "pLambdaS3KeyPrefix": "${pQuickstartS3KeyPrefix}" + "pLambdaS3BucketName": { + "Ref": "pQuickstartS3BucketName" + }, + "pLambdaS3KeyPrefix": { + "Ref": "pQuickstartS3KeyPrefix" + } }, "Tags": [ { diff --git a/cft-factory/gen_ha_iapp_cft.py b/cft-factory/gen_ha_iapp_cft.py index 0998f97..b84f4da 100644 --- a/cft-factory/gen_ha_iapp_cft.py +++ b/cft-factory/gen_ha_iapp_cft.py @@ -42,7 +42,7 @@ custom = template.add_resource(CustomResource( 'CustomLambdaExec', - ServiceToken=GetAtt('HAiApp', 'Arn'), + ServiceToken=GetAtt('InstallLambda', 'Arn'), mgmt_ip=Ref('pBigIPMgmt'), iapp_url=Ref('piAppUrl'), route_table_id=Ref('pBigIPRouteTableId'), From d35b4406ec6a0f41ec7bfcc6b207390ed0ce9827 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 12:18:10 -0500 Subject: [PATCH 46/61] added script to update the stack --- update.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 update.sh diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..580cc89 --- /dev/null +++ b/update.sh @@ -0,0 +1,14 @@ +if [ -f "./load-keys.sh" ]; then source "./load-keys.sh"; fi +if [ -z ${stackName} ]; then echo "stackName is unset" && exit 1; fi +if [ -z ${sshKeyName} ]; then echo "sshKeyName is unset" && exit 1; fi +if [ -z ${licenseKey1} ]; then echo "licenseKey1 is unset" && exit 1; fi +if [ -z ${licenseKey2} ]; then echo "licenseKey2 is unset" && exit 1; fi +if [ -z ${licenseKey3} ]; then echo "licenseKey3 is unset" && exit 1; fi +if [ -z ${licenseKey4} ]; then echo "licenseKey4 is unset" && exit 1; fi +if [ -z ${S3Key} ]; then export s3Key="master"; fi + +aws cloudformation update-stack \ +--stack-name $stackName \ +--template-body file://./aws-quickstart-scca-main-same-net.json \ +--capabilities CAPABILITY_IAM \ +--parameters ParameterKey=licenseKey1,ParameterValue=$licenseKey1 ParameterKey=licenseKey2,ParameterValue=$licenseKey2 ParameterKey=licenseKey3,ParameterValue=$licenseKey3 ParameterKey=licenseKey4,ParameterValue=$licenseKey4 ParameterKey=pBaselineCompliance,ParameterValue="Enterprise" ParameterKey=pQuickstartS3KeyPrefix,ParameterValue="$S3Key" ParameterKey=sshKey,ParameterValue="$sshKeyName" From 79c3a4a31f024eb4798f8aa92843ea6ff478a326 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 13:44:26 -0500 Subject: [PATCH 47/61] fix lambda reference --- aws-quickstart-scca-main-same-net.json | 11 +++++++++-- cft-factory/Makefile | 6 +++++- cft-factory/gen_ha_iapp_cft.py | 13 ++++++++++--- cft-factory/gen_install_lambda_cft.py | 9 ++++++++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/aws-quickstart-scca-main-same-net.json b/aws-quickstart-scca-main-same-net.json index df45995..40b100a 100644 --- a/aws-quickstart-scca-main-same-net.json +++ b/aws-quickstart-scca-main-same-net.json @@ -581,6 +581,9 @@ ] }, "Parameters": { + "pPrefix": { + "Fn::Sub": "${AWS::StackName}" + }, "pLambdaS3BucketName": { "Ref": "pQuickstartS3BucketName" }, @@ -603,7 +606,6 @@ }, "HAiAppTier1": { "Type": "AWS::CloudFormation::Stack", - "DependsOn": "F5BIGIP1", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -620,6 +622,9 @@ ] }, "Parameters": { + "pPrefix": { + "Fn::Sub": "${AWS::StackName}" + }, "pBigIPMgmt": { "Fn::GetAtt": [ "F5BIGIP1", @@ -655,7 +660,6 @@ }, "HAiAppTier2": { "Type": "AWS::CloudFormation::Stack", - "DependsOn": "F5BIGIP2", "Properties": { "TemplateURL": { "Fn::Sub": [ @@ -672,6 +676,9 @@ ] }, "Parameters": { + "pPrefix": { + "Fn::Sub": "${AWS::StackName}" + }, "pBigIPMgmt": { "Fn::GetAtt": [ "F5BIGIP2", diff --git a/cft-factory/Makefile b/cft-factory/Makefile index 33a943b..1fa18e5 100644 --- a/cft-factory/Makefile +++ b/cft-factory/Makefile @@ -7,4 +7,8 @@ install: cft: pipenv run python gen_ha_iapp_cft.py - pipenv run python gen_install_lambda_cft.py \ No newline at end of file + pipenv run python gen_install_lambda_cft.py + +validate: + pipenv run cfn-lint ha_iapp.json + pipenv run cfn-lint install_lambda.json \ No newline at end of file diff --git a/cft-factory/gen_ha_iapp_cft.py b/cft-factory/gen_ha_iapp_cft.py index b84f4da..121af27 100644 --- a/cft-factory/gen_ha_iapp_cft.py +++ b/cft-factory/gen_ha_iapp_cft.py @@ -1,4 +1,4 @@ -from troposphere import GetAtt, Join, Ref, Template, Parameter +from troposphere import GetAtt, Join, Ref, Template, Parameter, Sub from troposphere.cloudformation import AWSCustomObject, CustomResource from troposphere.awslambda import Code, Function @@ -8,6 +8,12 @@ template = Template() # add parameters +pPrefix = template.add_parameter(Parameter( + 'pPrefix', + Description='Lambda Function Name Prefix', + Type='String' +)) + piAppUrl = template.add_parameter(Parameter( 'piAppUrl', Description='URL to download the BIG-IP AWS HA iApp', @@ -29,7 +35,7 @@ pBigIPInterface = template.add_parameter(Parameter( 'pBigIPInterface', - Description='Tier 1 BIG-IP interface for HA', + Description='BIG-IP interface for HA', Type='String' )) @@ -42,7 +48,8 @@ custom = template.add_resource(CustomResource( 'CustomLambdaExec', - ServiceToken=GetAtt('InstallLambda', 'Arn'), + # ServiceToken=GetAtt('install-lambda-test-ha-iapp', 'Arn'), + ServiceToken=Sub('arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${pPrefix}-ha-iapp'), mgmt_ip=Ref('pBigIPMgmt'), iapp_url=Ref('piAppUrl'), route_table_id=Ref('pBigIPRouteTableId'), diff --git a/cft-factory/gen_install_lambda_cft.py b/cft-factory/gen_install_lambda_cft.py index 39cad02..5413ce9 100644 --- a/cft-factory/gen_install_lambda_cft.py +++ b/cft-factory/gen_install_lambda_cft.py @@ -1,4 +1,4 @@ -from troposphere import GetAtt, Join, Ref, Template, Parameter +from troposphere import GetAtt, Join, Ref, Template, Parameter, Sub from troposphere.iam import Role from troposphere.awslambda import Code, Function @@ -8,6 +8,12 @@ template = Template() # add parameters +pPrefix = template.add_parameter(Parameter( + 'pPrefix', + Description='Lambda Function Name Prefix', + Type='String' +)) + pLambdaS3BucketName = template.add_parameter(Parameter( 'pLambdaS3BucketName', Description='S3 bucket where lambda code is stored', @@ -42,6 +48,7 @@ ha_lambda = template.add_resource(Function( 'HAiApp', + FunctionName= Sub('${pPrefix}-ha-iapp'), Handler= 'index.lambda_handler', Role= GetAtt('LambdaRole', 'Arn'), Code= Code( From 0026ad5e6037acbd8135a0c2cc5d80c3996be8af Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 14:15:34 -0500 Subject: [PATCH 48/61] fix import error for cfnresponse --- lambda/ha_iapp/Makefile | 2 +- lambda/ha_iapp/index.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile index cbe918f..4217599 100644 --- a/lambda/ha_iapp/Makefile +++ b/lambda/ha_iapp/Makefile @@ -12,4 +12,4 @@ zip: chmod 755 $$(find ./ -type d) zip -FSr ha_iapp.zip ./* pipenv run pip uninstall -r requirements.txt -y - rm -rf requirements.txt f5_sca_libs* bin ch* dev* certifi* f5* ic* id* re* setup* six* url* \ No newline at end of file + rm -rf requirements.txt f5_sca_libs* bin ch* dev* certifi* f5* ic* id* re* setup* six* url* boto* cfn* date* doc* jme* python_dateutil* \ No newline at end of file diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py index e9f093e..083eadd 100644 --- a/lambda/ha_iapp/index.py +++ b/lambda/ha_iapp/index.py @@ -3,6 +3,7 @@ import boto3 import json import logging +import cfnresponse def lambda_handler(event, context): print("Received event: " + json.dumps(event, indent=2)) From b1c761b998c6620533c91d1c23f87c47e89b126c Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 14:30:40 -0500 Subject: [PATCH 49/61] fix timeout issue when cfnreponse is not called --- lambda/ha_iapp/index.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py index 083eadd..970e80d 100644 --- a/lambda/ha_iapp/index.py +++ b/lambda/ha_iapp/index.py @@ -6,7 +6,9 @@ import cfnresponse def lambda_handler(event, context): - print("Received event: " + json.dumps(event, indent=2)) + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.info('Received event: ' + json.dumps(event, indent=2)) if event['RequestType'] == 'Create': # obtain required variables @@ -19,19 +21,27 @@ def lambda_handler(event, context): if not mgmt_ip or not iapp_url or not route_table_id or not interface or not s3_bucket: raise Exception(f'ERROR: missing required event attributes {json.dumps(event, indent=2)} ') - # obtain BIG-IP Password - s3_client = boto3.client('s3') - pwd = password.get(s3_client, s3_bucket) - if not pwd: - raise Exception('ERROR: unable to obtain BIG-IP password') + # make sure we always return a reponse to CFN + try: + # obtain BIG-IP Password + s3_client = boto3.client('s3') + pwd = password.get(s3_client, s3_bucket) + if not pwd: + raise Exception('ERROR: unable to obtain BIG-IP password') - # deploy and configure ha_iapp - client = bigip.client(mgmt_ip, 'admin', pwd) - bigip.install_iapp(client, iapp_url) - iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') - bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) + # deploy and configure ha_iapp + client = bigip.client(mgmt_ip, 'admin', pwd) + bigip.install_iapp(client, iapp_url) + iapp_name = iapp_url.split('/')[-1].replace('.tmpl', '') + bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) - cfnresponse.send(event, context, cfnresponse.SUCCESS, True) + logger.info('Successfully updated the HA iApp') + cfnresponse.send(event, context, cfnresponse.SUCCESS, True) + return + + except Exception: + logger.exception('Signaling failure to CloudFormation.') + cfnresponse.send(event, context, cfnresponse.FAILED, {}) if event['RequestType'] == 'Delete': cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) \ No newline at end of file From e01a1a87aa77540cfdcaa5fd91b87a0f17d8d945 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 14:42:20 -0500 Subject: [PATCH 50/61] added response data for cfnreponse on Create event --- lambda/ha_iapp/index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lambda/ha_iapp/index.py b/lambda/ha_iapp/index.py index 970e80d..be473a2 100644 --- a/lambda/ha_iapp/index.py +++ b/lambda/ha_iapp/index.py @@ -36,9 +36,10 @@ def lambda_handler(event, context): bigip.cfg_ha_iapp(client, iapp_name, route_table_id, interface) logger.info('Successfully updated the HA iApp') - cfnresponse.send(event, context, cfnresponse.SUCCESS, True) + response = {'Response': 'true'} + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) return - + except Exception: logger.exception('Signaling failure to CloudFormation.') cfnresponse.send(event, context, cfnresponse.FAILED, {}) From 7f4dd74eb0d6edb1c8669b221eb3e90b14eeb6c8 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 16:28:12 -0500 Subject: [PATCH 51/61] add readme for cft-factory --- cft-factory/README.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 cft-factory/README.rst diff --git a/cft-factory/README.rst b/cft-factory/README.rst new file mode 100644 index 0000000..f6b2853 --- /dev/null +++ b/cft-factory/README.rst @@ -0,0 +1,23 @@ +cft-factory +======= + +Python funtions to generate Cloudformation Templates. + +Development +----------- + +1. Ensure ``Python3`` is installed as well as ``pip3`` +2. Clone repository: ``git@github.com:mikeoleary/f5-sca-securitystack.git`` +3. Install dev dependencies ``make install`` +4. Activate virtualenv: ``pipenv shell`` +5. Add your python file to the cft and validate sections of the makefile: + +Running Validation Tests +------------- + +Run tests locally using ``make cft && make validate`` if virtualenv is active: + +:: + + $ make cft && make validate + From 1c74eea919140d2882e2ed8789eb6a19f62e6f29 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 16:30:43 -0500 Subject: [PATCH 52/61] update ha_iapp readme and default make command now points to zip --- lambda/ha_iapp/Makefile | 2 +- lambda/ha_iapp/README.rst | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lambda/ha_iapp/Makefile b/lambda/ha_iapp/Makefile index 4217599..fbd2a7c 100644 --- a/lambda/ha_iapp/Makefile +++ b/lambda/ha_iapp/Makefile @@ -1,6 +1,6 @@ .PHONY: install test wheel -default: test +default: zip install: pipenv install --dev --skip-lock diff --git a/lambda/ha_iapp/README.rst b/lambda/ha_iapp/README.rst index 804e72f..8aed50e 100644 --- a/lambda/ha_iapp/README.rst +++ b/lambda/ha_iapp/README.rst @@ -11,17 +11,17 @@ Development 3. Install dev dependencies ``make install`` 4. Activate virtualenv: ``pipenv shell`` -Running Tests -------------- +Building Lambda ZIP file +------------------------- -Run tests locally using ``make`` if virtualenv is active: +Zip the lambda function so it can be uploaded to AWS using ``make zip`` if virtualenv is active: :: - $ make + $ make zip If virtualenv isn't active then use: :: - $ pipenv run make + $ pipenv run make zip \ No newline at end of file From fc8e4dd5b03a5e8c14555af6cb476b27f3887c70 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 16:40:33 -0500 Subject: [PATCH 53/61] added readme to lambda folder --- lambda/README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 lambda/README.rst diff --git a/lambda/README.rst b/lambda/README.rst new file mode 100644 index 0000000..e766ad8 --- /dev/null +++ b/lambda/README.rst @@ -0,0 +1,10 @@ +Lambda Function +=============== + +Lambda functions to assist in the deployment and configuration of the Secure Cloud Architecture + + +Development Pipeline +-------------------- + +When integrating with AWS Codepipeline use the ``buildspecs.yml`` in this folder to test the python and deploy the lambda artifacts. \ No newline at end of file From 89f2f2811abeaf3a150fc759ab92b2b099b458b1 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 21:05:14 -0500 Subject: [PATCH 54/61] automate pipeline build, move buildspec to root of repo --- lambda/buildspec.yml => buildspec.yml | 0 deploy-pipeline.template | 264 ++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) rename lambda/buildspec.yml => buildspec.yml (100%) create mode 100644 deploy-pipeline.template diff --git a/lambda/buildspec.yml b/buildspec.yml similarity index 100% rename from lambda/buildspec.yml rename to buildspec.yml diff --git a/deploy-pipeline.template b/deploy-pipeline.template new file mode 100644 index 0000000..625f552 --- /dev/null +++ b/deploy-pipeline.template @@ -0,0 +1,264 @@ +Parameters: + BranchName: + Description: GitHub branch name + Type: String + Default: master + RepositoryName: + Description: GitHub repository name + Type: String + Default: f5-sca-securitystack + GitHubOwner: + Type: String + OAuthSecretName: + Description: Name of OAuth Token in Secrets Manager + Type: String +Resources: + CodePipelineArtifactStoreBucket: + Type: 'AWS::S3::Bucket' + CodePipelineArtifactStoreBucketPolicy: + Type: 'AWS::S3::BucketPolicy' + Properties: + Bucket: !Ref CodePipelineArtifactStoreBucket + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: DenyUnEncryptedObjectUploads + Effect: Deny + Principal: '*' + Action: 's3:PutObject' + Resource: !Join + - '' + - - !GetAtt + - CodePipelineArtifactStoreBucket + - Arn + - /* + Condition: + StringNotEquals: + 's3:x-amz-server-side-encryption': 'aws:kms' + - Sid: DenyInsecureConnections + Effect: Deny + Principal: '*' + Action: 's3:*' + Resource: !Join + - '' + - - !GetAtt + - CodePipelineArtifactStoreBucket + - Arn + - /* + Condition: + Bool: + 'aws:SecureTransport': false + AppPipelineWebhook: + Type: 'AWS::CodePipeline::Webhook' + Properties: + Authentication: GITHUB_HMAC + AuthenticationConfiguration: + SecretToken: "Null" + Filters: + - JsonPath: $.ref + MatchEquals: 'refs/heads/{Branch}' + TargetPipeline: !Ref AppPipeline + TargetAction: SourceAction + Name: AppPipelineWebhook + TargetPipelineVersion: !GetAtt + - AppPipeline + - Version + RegisterWithThirdParty: true + AppPipeline: + Type: 'AWS::CodePipeline::Pipeline' + Properties: + Name: !Sub '${AWS::StackName}-pipeline' + RoleArn: !GetAtt + - CodePipelineServiceRole + - Arn + Stages: + - Name: Source + Actions: + - Name: SourceAction + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: "1" + Provider: GitHub + OutputArtifacts: + - Name: SourceOutput + Configuration: + Owner: !Ref GitHubOwner + Repo: !Ref RepositoryName + Branch: !Ref BranchName + OAuthToken: !Sub "{{resolve:secretsmanager:${OAuthSecretName}:SecretString:GitHubPersonalAccessToken}}" + PollForSourceChanges: false + RunOrder: 1 + - Name: Test + Actions: + - Name: CodeBuild + InputArtifacts: + - Name: SourceOutput + ActionTypeId: + Category: Build + Owner: AWS + Version: "1" + Provider: CodeBuild + Configuration: + ProjectName: !GetAtt [BuildProject, Arn ] + PrimarySource: SourceOutput + OutputArtifacts: + - Name: f5_sca_libs + RunOrder: 1 + ArtifactStore: + Type: S3 + Location: !Ref CodePipelineArtifactStoreBucket + CodePipelineServiceRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: !Sub '${AWS::StackName}-CodePipelineRole' + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - codepipeline.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: !Sub '${AWS::StackName}-CodePipelinePolicy' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'codecommit:CancelUploadArchive' + - 'codecommit:GetBranch' + - 'codecommit:GetCommit' + - 'codecommit:GetUploadArchiveStatus' + - 'codecommit:UploadArchive' + Resource: '*' + - Effect: Allow + Action: + - 'codedeploy:CreateDeployment' + - 'codedeploy:GetApplicationRevision' + - 'codedeploy:GetDeployment' + - 'codedeploy:GetDeploymentConfig' + - 'codedeploy:RegisterApplicationRevision' + Resource: '*' + - Effect: Allow + Action: + - 'codebuild:BatchGetBuilds' + - 'codebuild:StartBuild' + Resource: '*' + - Effect: Allow + Action: + - 'devicefarm:ListProjects' + - 'devicefarm:ListDevicePools' + - 'devicefarm:GetRun' + - 'devicefarm:GetUpload' + - 'devicefarm:CreateUpload' + - 'devicefarm:ScheduleRun' + Resource: '*' + - Effect: Allow + Action: + - 'lambda:InvokeFunction' + - 'lambda:ListFunctions' + Resource: '*' + - Effect: Allow + Action: + - 'iam:PassRole' + Resource: '*' + - Effect: Allow + Action: + - 'elasticbeanstalk:*' + - 'ec2:*' + - 'elasticloadbalancing:*' + - 'autoscaling:*' + - 'cloudwatch:*' + - 's3:*' + - 'sns:*' + - 'cloudformation:*' + - 'rds:*' + - 'sqs:*' + - 'ecs:*' + Resource: '*' + CodeBuildServiceRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: !Sub '${AWS::StackName}-CodeBuildRole' + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - codebuild.amazonaws.com + Action: 'sts:AssumeRole' + Path: /service-role/ + Policies: + - PolicyName: !Sub '${AWS::StackName}-CodeBuildPolicy' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'cloudformation:Get*' + - 'cloudformation:Describe*' + - 'cloudformation:List*' + Resource: '*' + - Effect: Allow + Action: + - "codecommit:ListBranches" + - "codecommit:ListRepositories" + - "codecommit:BatchGetRepositories" + - "codecommit:Get*" + - "codecommit:GitPull" + Resource: '*' + - Effect: Allow + Action: + - "ec2:Describe*" + - "cloudformation:ValidateTemplate" + - "elasticloadbalancing:Describe*" + - "autoscaling:Describe*" + - "iam:Get*" + - "iam:List*" + - "logs:Describe*" + - "logs:Get*" + - "tag:Get*" + Resource: '*' + - Effect: "Allow" + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Resource: + - Fn::Sub: arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/* + - Effect: "Allow" + Action: + - "s3:PutObject" + - "s3:GetObject" + - "s3:GetObjectVersion" + - "s3:ListBucket" + Resource: + - !Sub 'arn:aws:s3:::codepipeline-${AWS::Region}-*' + - !Sub 'arn:aws:s3:::${CodePipelineArtifactStoreBucket}/*' + - !Sub 'arn:aws:s3:::${CodePipelineArtifactStoreBucket}' + BuildProject: + Type: AWS::CodeBuild::Project + Properties: + Name: !Sub '${AWS::StackName}-BuildProject' + Description: Build to validate and copy CFN templates + ServiceRole: !GetAtt [CodeBuildServiceRole, Arn ] + Artifacts: + Type: S3 + Location: + Ref: CodePipelineArtifactStoreBucket + Name: !Sub '${AWS::StackName}-BuildCFN' + BadgeEnabled: true + Environment: + Type: LINUX_CONTAINER + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/amazonlinux2-x86_64-standard:1.0 + Source: + Auth: + Type: OAUTH + Location: !Sub 'https://github.com/${GitHubOwner}/${RepositoryName}' + Type: GITHUB + TimeoutInMinutes: 30 From 92a294c320f7df869f3d89e667c0abc073a404fc Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 22:17:08 -0500 Subject: [PATCH 55/61] add s3 deployment stage --- deploy-pipeline.template | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/deploy-pipeline.template b/deploy-pipeline.template index 625f552..97eb9b7 100644 --- a/deploy-pipeline.template +++ b/deploy-pipeline.template @@ -12,6 +12,13 @@ Parameters: OAuthSecretName: Description: Name of OAuth Token in Secrets Manager Type: String + S3Bucket: + Description: S3 bucket to deploy lambda and CFN templates into + Type: String + Default: f5-sca-securitystack + S3Key: + Description: S3 bucket folder to deploy into + Type: String Resources: CodePipelineArtifactStoreBucket: Type: 'AWS::S3::Bucket' @@ -105,6 +112,22 @@ Resources: OutputArtifacts: - Name: f5_sca_libs RunOrder: 1 + - Name: Deploy + Actions: + - Name: DeployArtifacts + InputArtifacts: + - Name: f5_sca_libs + ActionTypeId: + Category: Deploy + Owner: AWS + Version: "1" + Provider: "S3" + Configuration: + BucketName: !Ref S3Bucket + ObjectKey: !Ref S3Key + Extract: 'true' + + ArtifactStore: Type: S3 Location: !Ref CodePipelineArtifactStoreBucket From 4a902cee8f1ecbf4763c8693e99c59cb6dfb5907 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 22:28:38 -0500 Subject: [PATCH 56/61] add deployment action for source repo until issue #49 is addressed --- deploy-pipeline.template | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy-pipeline.template b/deploy-pipeline.template index 97eb9b7..186003b 100644 --- a/deploy-pipeline.template +++ b/deploy-pipeline.template @@ -114,6 +114,18 @@ Resources: RunOrder: 1 - Name: Deploy Actions: + - Name: DeploySource + InputArtifacts: + - Name: SourceOutput + ActionTypeId: + Category: Deploy + Owner: AWS + Version: "1" + Provider: "S3" + Configuration: + BucketName: !Ref S3Bucket + ObjectKey: !Ref S3Key + Extract: 'true' - Name: DeployArtifacts InputArtifacts: - Name: f5_sca_libs From 146b78423d5d658201a514c0123c7b0d8d88d7e4 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Tue, 27 Aug 2019 22:39:23 -0500 Subject: [PATCH 57/61] added developer notes to the readme --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 7c70e91..5155893 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,18 @@ Third, BIG-IP tier 1 and tier 2 are deployed Fourth, routes are replaced utilizing lambda functions since it's not natively allowed Lastly, the AS3 Update template updates AS3 + +## Development +This project uses the AWS CloudPipeline to build the require Lambda functions as well as generating some of the CFN Templates. + +To start developing against this project please follow the below procedures: + +1) Create a GitHub [Personal Access Token](https://docs.aws.amazon.com/codepipeline/latest/userguide/GitHub-create-personal-token-CLI.html) +2) Add the GitHub PAT to your AWS Secrets Manager. **Note:** ensure the key uses the value GitHubPersonalAccessToken +3) Deploy the deploy-pipeline.template CloudFormation Template + 1. BranchName: the branch name of your Git repository - usually master unless you created a branch for development. + 2. GitHubOwner: the GitHub account owner for the repository. This is used to build the GitHub URL to access the repository. + 3. OAuthSecretName: the name of your AWS Secrets Manager object that stores the GitHub PAT + 4. RepositoryName: the Git repository name. This is used to build the GitHub URL to access the repository. + 5. S3Bucket: the S3 bucket you want the CFN templates and lambda files installed to. **Note:** this bucket must already exists. + 6. S3Key: the directory inside your S3 bucket that will contain the deployment artifacts. From 9984cdaf74fe8c84246663109cfc652ce6957a06 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 28 Aug 2019 07:01:36 -0500 Subject: [PATCH 58/61] remove static ha_iapp CFN template --- BIG-IP/ha_iapp.json | 178 -------------------------------------------- 1 file changed, 178 deletions(-) delete mode 100644 BIG-IP/ha_iapp.json diff --git a/BIG-IP/ha_iapp.json b/BIG-IP/ha_iapp.json deleted file mode 100644 index 77c377f..0000000 --- a/BIG-IP/ha_iapp.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Parameters": { - "pVpcStackName": { - "Type": "String", - "Default": "SCCA-Demo", - "Description": "VPC Stack used for referencing exported resources" - }, - "pBigIPTier1Mgmt": { - "Type": "String", - "Default": "", - "Description": "BIGIP Management IP address for Tier1" - }, - "pBigIPTier2Mgmt": { - "Type": "String", - "Default": "", - "Description": "BIGIP Management IP address for Tier2" - }, - "pIAppUrl": { - "Type": "String", - "Default": "https://raw.githubusercontent.com/F5Networks/f5-aws-cloudformation/v3.1.0/iApps/f5.aws_advanced_ha.v1.4.0rc3.tmpl", - "Description": "URL to download BIG-IP AWS HA iApp" - }, - "pBigIPTier1RouteTableId": { - "Type": "String", - "Default": "", - "Description": "Route Table ID for Tier1 HA iApp" - }, - "pBigIPTier2RouteTableId": { - "Type": "String", - "Default": "", - "Description": "Route Table ID for Tier2 HA iApp" - }, - "pBigIPTier1Interface": { - "Type": "String", - "Default": "/Common/internal", - "Description": "Interface for Tier1 HA iApp" - }, - "pBigIPTier2Interface": { - "Type": "String", - "Default": "/Common/external", - "Description": "Interface for Tier2 HA iApp" - }, - "pBigIPTier1S3Bucket": { - "Type": "String", - "Default": "", - "Description": "S3 Bucket where BIG-IP password is stored" - }, - "pBigIPTier2S3Bucket": { - "Type": "String", - "Default": "", - "Description": "S3 Bucket where BIG-IP password is stored" - }, - "pQuickstartS3BucketName": { - "Type": "String", - "Default": "f5-sca-securitystack", - "Description": "S3 bucket name for the Quick Start assets. Quick Start bucket name can include numbers, lowercase letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-)." - }, - "pQuickstartS3KeyPrefix": { - "Type": "String", - "Default": "master", - "Description": "Quick Start key prefix can include numbers, lowercase letters, uppercase letters, hyphens (-), and forward slash (/)." - } - }, - "Resources": { - "IamRole": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - }, - "Action": [ - "sts:AssumeRole" - ] - } - ] - }, - "Path": "/", - "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/AmazonVPCFullAccess", - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" - ] - } - }, - "CustomDeployHAiAppTier1": { - "Type": "Custom::DeployHAiApp", - "Version": "1.0", - "Properties": { - "ServiceToken": { - "Fn:GetAtt": [ - "LambdaHAiApp", - "Arn" - ] - }, - "mgmt_ip": { - "Ref": "pBigIPTier1Mgmt" - }, - "iapp_url": { - "Ref": "pIAppUrl" - }, - "route_table_id": { - "Ref": "pBigIPTier1RouteTableId" - }, - "interface": { - "Ref": "pBigIPTier1Interface" - }, - "s3_bucket": { - "Ref": "pBigIPTier1S3Bucket" - } - } - }, - "CustomDeployHAiAppTier2": { - "Type": "Custom::DeployHAiApp", - "Version": "1.0", - "Properties": { - "ServiceToken": { - "Fn:GetAtt": [ - "LambdaHAiApp", - "Arn" - ] - }, - "mgmt_ip": { - "Ref": "pBigIPTier2Mgmt" - }, - "iapp_url": { - "Ref": "pIAppUrl" - }, - "route_table_id": { - "Ref": "pBigIPTier2RouteTableId" - }, - "interface": { - "Ref": "pBigIPTier2Interface" - }, - "s3_bucket": { - "Ref": "pBigIPTier2S3Bucket" - } - } - }, - "LambdaHAiApp": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Handler": "index.lambda_handler", - "Role": { - "Fn::GetAtt": [ - "IamRole", - "Arn" - ] - }, - "Runtime": "python3.6", - "Timeout": 30, - "Code": { - "S3Bucket": { - "Ref": "pQuickstartS3BucketName" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Ref": "pQuickstartS3KeyPrefix" - }, - "/ha_iapp.zip" - ] - ] - } - } - } - } - } -} \ No newline at end of file From 2a469736ed2303db3fb61a7722adbeda7b201de3 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 28 Aug 2019 07:18:23 -0500 Subject: [PATCH 59/61] fix formating differences --- deploy-pipeline.template | 150 +++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/deploy-pipeline.template b/deploy-pipeline.template index 186003b..b3136b8 100644 --- a/deploy-pipeline.template +++ b/deploy-pipeline.template @@ -21,9 +21,9 @@ Parameters: Type: String Resources: CodePipelineArtifactStoreBucket: - Type: 'AWS::S3::Bucket' + Type: "AWS::S3::Bucket" CodePipelineArtifactStoreBucketPolicy: - Type: 'AWS::S3::BucketPolicy' + Type: "AWS::S3::BucketPolicy" Properties: Bucket: !Ref CodePipelineArtifactStoreBucket PolicyDocument: @@ -31,39 +31,39 @@ Resources: Statement: - Sid: DenyUnEncryptedObjectUploads Effect: Deny - Principal: '*' - Action: 's3:PutObject' + Principal: "*" + Action: "s3:PutObject" Resource: !Join - - '' + - "" - - !GetAtt - CodePipelineArtifactStoreBucket - Arn - /* Condition: StringNotEquals: - 's3:x-amz-server-side-encryption': 'aws:kms' + "s3:x-amz-server-side-encryption": "aws:kms" - Sid: DenyInsecureConnections Effect: Deny - Principal: '*' - Action: 's3:*' + Principal: "*" + Action: "s3:*" Resource: !Join - - '' + - "" - - !GetAtt - CodePipelineArtifactStoreBucket - Arn - /* Condition: Bool: - 'aws:SecureTransport': false + "aws:SecureTransport": false AppPipelineWebhook: - Type: 'AWS::CodePipeline::Webhook' + Type: "AWS::CodePipeline::Webhook" Properties: Authentication: GITHUB_HMAC AuthenticationConfiguration: SecretToken: "Null" Filters: - JsonPath: $.ref - MatchEquals: 'refs/heads/{Branch}' + MatchEquals: "refs/heads/{Branch}" TargetPipeline: !Ref AppPipeline TargetAction: SourceAction Name: AppPipelineWebhook @@ -72,9 +72,9 @@ Resources: - Version RegisterWithThirdParty: true AppPipeline: - Type: 'AWS::CodePipeline::Pipeline' + Type: "AWS::CodePipeline::Pipeline" Properties: - Name: !Sub '${AWS::StackName}-pipeline' + Name: !Sub "${AWS::StackName}-pipeline" RoleArn: !GetAtt - CodePipelineServiceRole - Arn @@ -125,7 +125,7 @@ Resources: Configuration: BucketName: !Ref S3Bucket ObjectKey: !Ref S3Key - Extract: 'true' + Extract: "true" - Name: DeployArtifacts InputArtifacts: - Name: f5_sca_libs @@ -137,16 +137,16 @@ Resources: Configuration: BucketName: !Ref S3Bucket ObjectKey: !Ref S3Key - Extract: 'true' + Extract: "true" ArtifactStore: Type: S3 Location: !Ref CodePipelineArtifactStoreBucket CodePipelineServiceRole: - Type: 'AWS::IAM::Role' + Type: "AWS::IAM::Role" Properties: - RoleName: !Sub '${AWS::StackName}-CodePipelineRole' + RoleName: !Sub "${AWS::StackName}-CodePipelineRole" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -154,70 +154,70 @@ Resources: Principal: Service: - codepipeline.amazonaws.com - Action: 'sts:AssumeRole' + Action: "sts:AssumeRole" Path: / Policies: - - PolicyName: !Sub '${AWS::StackName}-CodePipelinePolicy' + - PolicyName: !Sub "${AWS::StackName}-CodePipelinePolicy" PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - - 'codecommit:CancelUploadArchive' - - 'codecommit:GetBranch' - - 'codecommit:GetCommit' - - 'codecommit:GetUploadArchiveStatus' - - 'codecommit:UploadArchive' - Resource: '*' + - "codecommit:CancelUploadArchive" + - "codecommit:GetBranch" + - "codecommit:GetCommit" + - "codecommit:GetUploadArchiveStatus" + - "codecommit:UploadArchive" + Resource: "*" - Effect: Allow Action: - - 'codedeploy:CreateDeployment' - - 'codedeploy:GetApplicationRevision' - - 'codedeploy:GetDeployment' - - 'codedeploy:GetDeploymentConfig' - - 'codedeploy:RegisterApplicationRevision' - Resource: '*' + - "codedeploy:CreateDeployment" + - "codedeploy:GetApplicationRevision" + - "codedeploy:GetDeployment" + - "codedeploy:GetDeploymentConfig" + - "codedeploy:RegisterApplicationRevision" + Resource: "*" - Effect: Allow Action: - - 'codebuild:BatchGetBuilds' - - 'codebuild:StartBuild' - Resource: '*' + - "codebuild:BatchGetBuilds" + - "codebuild:StartBuild" + Resource: "*" - Effect: Allow Action: - - 'devicefarm:ListProjects' - - 'devicefarm:ListDevicePools' - - 'devicefarm:GetRun' - - 'devicefarm:GetUpload' - - 'devicefarm:CreateUpload' - - 'devicefarm:ScheduleRun' - Resource: '*' + - "devicefarm:ListProjects" + - "devicefarm:ListDevicePools" + - "devicefarm:GetRun" + - "devicefarm:GetUpload" + - "devicefarm:CreateUpload" + - "devicefarm:ScheduleRun" + Resource: "*" - Effect: Allow Action: - - 'lambda:InvokeFunction' - - 'lambda:ListFunctions' - Resource: '*' + - "lambda:InvokeFunction" + - "lambda:ListFunctions" + Resource: "*" - Effect: Allow Action: - - 'iam:PassRole' - Resource: '*' + - "iam:PassRole" + Resource: "*" - Effect: Allow Action: - - 'elasticbeanstalk:*' - - 'ec2:*' - - 'elasticloadbalancing:*' - - 'autoscaling:*' - - 'cloudwatch:*' - - 's3:*' - - 'sns:*' - - 'cloudformation:*' - - 'rds:*' - - 'sqs:*' - - 'ecs:*' - Resource: '*' + - "elasticbeanstalk:*" + - "ec2:*" + - "elasticloadbalancing:*" + - "autoscaling:*" + - "cloudwatch:*" + - "s3:*" + - "sns:*" + - "cloudformation:*" + - "rds:*" + - "sqs:*" + - "ecs:*" + Resource: "*" CodeBuildServiceRole: - Type: 'AWS::IAM::Role' + Type: "AWS::IAM::Role" Properties: - RoleName: !Sub '${AWS::StackName}-CodeBuildRole' + RoleName: !Sub "${AWS::StackName}-CodeBuildRole" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -225,19 +225,19 @@ Resources: Principal: Service: - codebuild.amazonaws.com - Action: 'sts:AssumeRole' + Action: "sts:AssumeRole" Path: /service-role/ Policies: - - PolicyName: !Sub '${AWS::StackName}-CodeBuildPolicy' + - PolicyName: !Sub "${AWS::StackName}-CodeBuildPolicy" PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - - 'cloudformation:Get*' - - 'cloudformation:Describe*' - - 'cloudformation:List*' - Resource: '*' + - "cloudformation:Get*" + - "cloudformation:Describe*" + - "cloudformation:List*" + Resource: "*" - Effect: Allow Action: - "codecommit:ListBranches" @@ -245,7 +245,7 @@ Resources: - "codecommit:BatchGetRepositories" - "codecommit:Get*" - "codecommit:GitPull" - Resource: '*' + Resource: "*" - Effect: Allow Action: - "ec2:Describe*" @@ -257,7 +257,7 @@ Resources: - "logs:Describe*" - "logs:Get*" - "tag:Get*" - Resource: '*' + Resource: "*" - Effect: "Allow" Action: - "logs:CreateLogGroup" @@ -272,20 +272,20 @@ Resources: - "s3:GetObjectVersion" - "s3:ListBucket" Resource: - - !Sub 'arn:aws:s3:::codepipeline-${AWS::Region}-*' - - !Sub 'arn:aws:s3:::${CodePipelineArtifactStoreBucket}/*' - - !Sub 'arn:aws:s3:::${CodePipelineArtifactStoreBucket}' + - !Sub "arn:aws:s3:::codepipeline-${AWS::Region}-*" + - !Sub "arn:aws:s3:::${CodePipelineArtifactStoreBucket}/*" + - !Sub "arn:aws:s3:::${CodePipelineArtifactStoreBucket}" BuildProject: Type: AWS::CodeBuild::Project Properties: - Name: !Sub '${AWS::StackName}-BuildProject' + Name: !Sub "${AWS::StackName}-BuildProject" Description: Build to validate and copy CFN templates ServiceRole: !GetAtt [CodeBuildServiceRole, Arn ] Artifacts: Type: S3 Location: Ref: CodePipelineArtifactStoreBucket - Name: !Sub '${AWS::StackName}-BuildCFN' + Name: !Sub "${AWS::StackName}-BuildCFN" BadgeEnabled: true Environment: Type: LINUX_CONTAINER @@ -294,6 +294,6 @@ Resources: Source: Auth: Type: OAUTH - Location: !Sub 'https://github.com/${GitHubOwner}/${RepositoryName}' + Location: !Sub "https://github.com/${GitHubOwner}/${RepositoryName}" Type: GITHUB TimeoutInMinutes: 30 From 6c826f673d61bd4efd0fd40d21c61bf3a03fbffc Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 28 Aug 2019 08:44:48 -0500 Subject: [PATCH 60/61] fix apppipelinewebhook name to be unique --- deploy-pipeline.template | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/deploy-pipeline.template b/deploy-pipeline.template index b3136b8..d78ec2d 100644 --- a/deploy-pipeline.template +++ b/deploy-pipeline.template @@ -66,7 +66,7 @@ Resources: MatchEquals: "refs/heads/{Branch}" TargetPipeline: !Ref AppPipeline TargetAction: SourceAction - Name: AppPipelineWebhook + Name: !Sub '{$AWS::StackName}-AppPipelineWebhook' TargetPipelineVersion: !GetAtt - AppPipeline - Version @@ -121,7 +121,7 @@ Resources: Category: Deploy Owner: AWS Version: "1" - Provider: "S3" + Provider: S3 Configuration: BucketName: !Ref S3Bucket ObjectKey: !Ref S3Key @@ -133,13 +133,11 @@ Resources: Category: Deploy Owner: AWS Version: "1" - Provider: "S3" + Provider: S3 Configuration: BucketName: !Ref S3Bucket ObjectKey: !Ref S3Key Extract: "true" - - ArtifactStore: Type: S3 Location: !Ref CodePipelineArtifactStoreBucket From d27dea1ae12cc1e2078b1e798985e38620a32c10 Mon Sep 17 00:00:00 2001 From: Cody Green Date: Wed, 28 Aug 2019 08:46:28 -0500 Subject: [PATCH 61/61] fix sub error in template --- deploy-pipeline.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-pipeline.template b/deploy-pipeline.template index d78ec2d..a1be0ee 100644 --- a/deploy-pipeline.template +++ b/deploy-pipeline.template @@ -66,7 +66,7 @@ Resources: MatchEquals: "refs/heads/{Branch}" TargetPipeline: !Ref AppPipeline TargetAction: SourceAction - Name: !Sub '{$AWS::StackName}-AppPipelineWebhook' + Name: !Sub "${AWS::StackName}-AppPipelineWebhook" TargetPipelineVersion: !GetAtt - AppPipeline - Version