Add support to deployment in non-control environments

.project_automation/functional_tests/entrypoint.sh

@@ -43,6 +43,7 @@ run_test "launch-qradar-main-default"
 
 run_test "launch-qradar-main-both"
 
+#run_test "launch-qradar-main-both-nonct"
 ## Executing ash tool
 
 #find ${PROJECT_PATH} -name lambda.zip -exec rm -rf {} \;

.taskcat.yml

@@ -29,6 +29,24 @@ tests:
       - us-east-1
     template: templates/abi-enable-qradar-integration.yaml
 
+  launch-qradar-main-both-nonct:
+    parameters:
+      pSRASourceS3BucketName: $[taskcat_autobucket]
+      pSRAS3BucketRegion: $[taskcat_current_region]
+      pSRAStagingS3KeyPrefix: $[taskcat_project_name]
+      pConfigureCloudTrail: 'Yes'
+      pConfigureGuardDuty: 'Yes'
+      pDisableGuardDuty: 'Yes'
+      pControlTower: 'false'
+      pLogArchiveAccountId: $[taskcat_ssm_/nonct/log-archive-account-id]
+      pSecurityAccountId: $[taskcat_ssm_/nonct/audit-account-id]
+      pGovernedRegions: 'us-east-1,us-east-2'
+      pAdminRoleName: 'AWSCloudFormationStackSetAdministrationRole'
+      pExecRoleName: 'AWSCloudFormationStackSetExecutionRole'
+    regions:
+      - us-east-1
+    template: templates/abi-enable-qradar-integration.yaml
+
   # launch-qradar-main-only-ct:
   #   parameters:
   #     pSRASourceS3BucketName: $[taskcat_autobucket]

scripts/cleanup_config.json

@@ -3,6 +3,18 @@
     "Type" : "STACK",
     "Filter" : "tCaT-launch-qradar"
   },
+  {
+    "Type" : "STACK_SET",
+    "Filter" : "qradar-setup-iam-role-stack"
+  },
+  {
+    "Type" : "STACK_SET",
+    "Filter" : "qradar-cloudtrail-event-stack"
+  },
+  {
+    "Type" : "STACK_SET",
+    "Filter" : "qradar-guardduty-event-stack"
+  },
   {
     "Type" : "SSM_PARAMETER",
     "Filter" : "/sra/gd/",
@@ -39,7 +51,6 @@
     "Type": "S3_BUCKET",
     "Filter": "tcat-"
   },
-
   {
     "Type": "S3_BUCKET",
     "Filter": "sra-cloudtrail-staging-"
@@ -92,6 +103,9 @@
     "Type" : "CODE_BUILD",
     "Filter": "sra-codebuild-project"
   },
+  {
+    "Type" : "GUARDDUTY_DET"
+  },
   {
     "Type" : "STACK_SET",
     "Filter" : "sra-stackset-execution-role"

scripts/cleanup_config.py

@@ -30,26 +30,27 @@
                 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_IN_PROGRESS',
                 'UPDATE_ROLLBACK_FAILED', 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
                 'UPDATE_ROLLBACK_COMPLETE', 'REVIEW_IN_PROGRESS']
-def list_stacksets():
+
+def list_stacksets(context=CF):
     '''List all stacksets in the account'''
-    response = CF.list_stack_sets()
+    response = context.list_stack_sets()
     stacksets = response['Summaries']
     while response.get('NextToken'):
-        response = CF.list_stack_sets(NextToken=response['NextToken'])
+        response = context.list_stack_sets(NextToken=response['NextToken'])
         stacksets.extend(response['Summaries'])
     return stacksets
 
-def list_active_stackset_names():
+def list_active_stackset_names(context=CF):
     '''List all stackset names in the account'''
     cf_names = []
-    for cfn in list_stacksets():
+    for cfn in list_stacksets(context):
         if cfn['Status'] != 'DELETED':
             cf_names += [cfn['StackSetName']]
     return cf_names
 
-def list_stackset_names(filters=None):
+def list_stackset_names(context=CF, filters=None):
     '''List all stackset names in the account'''
-    cf_info = list_stacksets()
+    cf_info = list_stacksets(context)
     cf_names = []
     for cfn in cf_info:
         if cfn['Status'] != 'DELETED':
@@ -62,35 +63,35 @@ def list_stackset_names(filters=None):
 
     return cf_names
 
-def list_stackset_instances(stackset_name):
+def list_stackset_instances(context=CF, ss_name=None):
     '''List all stackset instances in the account'''
-    response = CF.list_stack_instances(StackSetName=stackset_name)
+    response = context.list_stack_instances(StackSetName=ss_name)
     stackinstances = response['Summaries']
     while response.get('NextToken'):
-        response = CF.list_stack_instances(StackSetName=stackset_name,
+        response = context.list_stack_instances(StackSetName=ss_name,
                                         NextToken=response['NextToken'])
         stackinstances.extend(response['Summaries'])
     return stackinstances
 
 def delete_stack_instances(stackset_name, retain_stacks=False):
     '''Delete all stackset instances in the account'''
-    stackinstances = list_stackset_instances(stackset_name)
+    stackinstances = list_stackset_instances(ss_name=stackset_name)
     for stackinstance in stackinstances:
         CF.delete_stack_instances(StackSetName=stackset_name,
                                 Regions=[stackinstance['Region']],
                                 RetainStacks=retain_stacks)
 
 def si_account_list(stackset_name):
     '''List all stackset instance accounts'''
-    stackinstances = list_stackset_instances(stackset_name)
+    stackinstances = list_stackset_instances(ss_name=stackset_name)
     stackinstance_names = []
     for stackinstance in stackinstances:
         stackinstance_names += [stackinstance['Account']]
     return stackinstance_names
 
 def si_region_list(stackset_name):
     '''List all stackset instance regions'''
-    stackinstances = list_stackset_instances(stackset_name)
+    stackinstances = list_stackset_instances(ss_name=stackset_name)
     stackinstance_regions = []
     for stackinstance in stackinstances:
         stackinstance_regions += [stackinstance['Region']]
@@ -127,15 +128,15 @@ def delete_all_stackinstances(stackset_name):
                                                  RetainStacks=False)
 
         loop = 1
-        while len(list_stackset_instances(stackset_name)) > 0 and loop < 30:
+        while len(list_stackset_instances(ss_name=stackset_name)) > 0 and loop < 30:
             sleep(10)
             loop += 1
 
     return response
 
 def delete_stacksets(filters):
     '''Delete all stacksets created by CfCT solution in the account'''
-    cf_names = list_stackset_names(filters)
+    cf_names = list_stackset_names(filters=filters)
     for cf_name in cf_names:
         op_info = delete_all_stackinstances(cf_name)
         op_id = op_info['OperationId']
@@ -151,11 +152,11 @@ def delete_stacksets(filters):
         CF.delete_stack_set(StackSetName=cf_name)
 
 def list_all_stacks():
-    '''List all stacks in the account with status other than DELETE_COMPLETE'''
-    response = CF.list_stacks(StackStatusFilter=VALID_STATUS)
+    '''List all stacks in the account'''
+    response = CF.list_stacks()
     stacks = response['StackSummaries']
     while response.get('NextToken'):
-        response = CF.list_stacks(StackStatusFilter=VALID_STATUS, NextToken=response['NextToken'])
+        response = CF.list_stacks(NextToken=response['NextToken'])
         stacks.extend(response['StackSummaries'])
     return stacks
 
@@ -245,29 +246,33 @@ def delete_s3_buckets(item):
                 else:
                     raise exe
 
-def list_all_parameters(ssm_session):
+def list_all_parameters(context):
     ''''List all parameters in the account'''
-    response = ssm_session.describe_parameters()
-    result = response['Parameters']
+    response = context.describe_parameters()
+    parameters = response['Parameters']
     while response.get('NextToken'):
-        response = ssm_session.describe_parameters(NextToken=response['NextToken'])
-        result.extend(response['Parameters'])
-
-    return result
+        response = context.describe_parameters(NextToken=response['NextToken'])
+        parameters.extend(response['Parameters'])
+    return parameters
 
 def delete_parameters(item):
     '''Delete all parameters created in the account'''
 
+    print(f"Recieved item: {item}")
     filters = item['Filter']
     (ssm_session, account, target) = get_client_session(item, 'ssm')
-    print(f"SSM action on {target} with filters: {filters}")
+    print(f"SSM action on {account}/{target} with filters: {filters}")
 
-    parameters = list_all_parameters(ssm_session)
+    parameters = list_all_parameters(context=ssm_session)
     for parameter in parameters:
         param_name = parameter['Name']
+        print(f"param_name: {param_name}")
         if param_name.startswith(filters):
             print(f"..Deleting parameter {param_name}.")
+            res = ssm_session.get_parameter(Name=param_name)['Parameter']['ARN']
+            print(f"res: {res}")
             ssm_session.delete_parameter(Name=param_name)
+            print(f"..Deleted parameter {param_name}.")
 
 def get_temp_credentials(aws_account, role_name='AWSControlTowerExecution'):
     '''
@@ -306,6 +311,7 @@ def establish_remote_session(account):
             aws_secret_access_key=sts_creds['SecretAccessKey'],
             aws_session_token=sts_creds['SessionToken']
             )
+            print(f"Established session for {account} with {role}")
             break
 
     return result
@@ -360,7 +366,7 @@ def delete_cw_logs(item):
 
     filters = item['Filter']
     (cw_session, account, target) = get_client_session(item, 'logs')
-    print(f"LOG GROUP action on {target} with filters: {filters}")
+    print(f"LOG GROUP action on {account}/{target} with filters: {filters}")
 
     log_groups = list_cw_lognames(context=cw_session)
     for log_group_name in log_groups:
@@ -408,7 +414,6 @@ def delete_detector():
                 print('Deleting GuardDuty Detector in %s', account['Id'])
                 gd_client.delete_detector(DetectorId=det_id)
 
-
 def list_cb_projects():
     '''
     List all CodeBuild projects
@@ -433,35 +438,57 @@ def delete_build_projects(filters='sra-codebuild-'):
             cb_session = SESSION.client('codebuild')
             cb_session.delete_project(name=project)
 
-def get_account_info(ss_name='AWSControlTowerLoggingResources'):
+def get_log_account_info(context, ss_name='AWSControlTowerLoggingResources'):
+    ''' List first stack instances in a stackset '''
+
+    cf_client = context.client('cloudformation')
+    if ss_name in list_active_stackset_names(cf_client):
+        instance = cf_client.list_stack_instances(StackSetName=ss_name)
+        account_id = instance['Summaries'][0]['Account']
+        for account in get_list_of_accounts():
+            if account['Id'] == account_id:
+                account_name = account['Name']
+    else:
+        account_id = get_account_id('Log Archive')
+        account_name = 'LogArchive'
+
+    return {'AccountName': account_name, 'AccountID': account_id}
+
+def get_audit_account_info(context, ss_name='AWSControlTowerLoggingResources'):
     '''
     List first stack instances in a stackset
     '''
-    result = None
 
-    if ss_name in list_active_stackset_names():
-        instance = CF.list_stack_instances(StackSetName=ss_name)
+    cf_client = context.client('cloudformation')
+    if ss_name in list_active_stackset_names(cf_client):
+        instance = cf_client.list_stack_instances(StackSetName=ss_name)
         account_id = instance['Summaries'][0]['Account']
         for account in get_list_of_accounts():
             if account['Id'] == account_id:
                 account_name = account['Name']
-        result = {'AccountName': account_name, 'AccountID': account_id}
-    return result
+    else:
+        account_id = get_account_id('Audit')
+        account_name = 'Audit'
+
+    return {'AccountName': account_name, 'AccountID': account_id}
 
 def get_client_session(item, client_name):
-    '''
-    Return a session for parent or child
-    '''
+    ''' Return a session for parent or child '''
 
     account = None
     if 'Account' in item:
         if item['Account'] in ACCOUNTS:
+            print(f"Using account {ACCOUNTS[item['Account']]} for {client_name}")
             account = get_account_id(ACCOUNTS[item['Account']])
+            print(f"Got account ID: {account}")
+
     if account:
+        print(f"Establishing session to : {account}")
         session = establish_remote_session(account)
         client_session = session.client(client_name)
         target = account
     else:
+        print(f"Using local session for {client_name}")
         client_session = boto3.client(client_name)
         target = STS.get_caller_identity()['Account']
 
@@ -474,13 +501,17 @@ def delete_iam_role(item):
 
     role_name = item['Filter']
     (iam_session, account, target) = get_client_session(item, 'iam')
-    print(f"IAM action on {target} with role_name: {role_name}")
+    print(f"IAM action on {account}/{target} with role_name: {role_name}")
 
     try:
         policies = iam_session.list_attached_role_policies(RoleName=role_name)
         for policy in policies['AttachedPolicies']:
             print(f"..Detaching policy {policy['PolicyArn']} from role {role_name}.")
             iam_session.detach_role_policy(RoleName=role_name, PolicyArn=policy['PolicyArn'])
+        policies = iam_session.list_role_policies(RoleName=role_name)
+        for policy in policies['PolicyNames']:
+            print(f"..Deleting inline policy {policy} from role {role_name}.")
+            iam_session.delete_role_policy(RoleName=role_name, PolicyName=policy)
         print(f"....Deleting role {role_name}.")
         iam_session.delete_role(RoleName=role_name)
     except Exception as exe:
@@ -520,23 +551,25 @@ def run_cleanup(config):
                                      description='Clear the configuration.')
     PARSER.add_argument("-C", "--config", default='cleanup_config.json',
                         help="Clear content from config")
+    PARSER.add_argument("-r", "--home_region", default='us-east-1',
+                        help="Control Tower home region")
 
     ARGS = PARSER.parse_args()
+    CT_HOME = ARGS.home_region
+
+    ACC_SESSION = boto3.session.Session(region_name=CT_HOME)
+    LOG_ACCT_NAME = 'Log Archive'
+    AUDIT_ACCT_NAME = 'Audit'
+
+    LOG_ACCT_INFO = get_log_account_info(context=ACC_SESSION)
+    AUDIT_ACCT_INFO = get_audit_account_info(ACC_SESSION)
 
-    LOG_ACCT_INFO = get_account_info('AWSControlTowerLoggingResources')
-    AUDIT_ACCT_INFO = get_account_info('AWSControlTowerSecurityResources')
     if LOG_ACCT_INFO:
         LOG_ACCT_NAME = LOG_ACCT_INFO['AccountName']
-    else:
-        LOG_ACCT_NAME = 'Log Archive'
-
     if AUDIT_ACCT_INFO:
         AUDIT_ACCT_NAME = AUDIT_ACCT_INFO['AccountName']
-    else:
-        AUDIT_ACCT_NAME = 'Audit'
 
     ACCOUNTS = {"log_account": LOG_ACCT_NAME, "audit": AUDIT_ACCT_NAME}
-    print('Recieved Account Info: %s', ACCOUNTS)
     CLEAR_CFG = ARGS.config
 
     if isfile(CLEAR_CFG):