Merge pull request #61 from aws-ia/devel

Enable scotsuite execution as part of functional tests
aws-ia · Jan 23, 2024 · 2d99972 · 2d99972
2 parents 4bbbfbd + 31e2d5b
commit 2d99972
Show file tree

Hide file tree

Showing 10 changed files with 76 additions and 49 deletions.
diff --git a/.project_automation/functional_tests/entrypoint.sh b/.project_automation/functional_tests/entrypoint.sh
@@ -5,6 +5,7 @@
 # managed and local tasks always use these variables for the project and project type path
 PROJECT_PATH=${BASE_PATH}/project
 PROJECT_TYPE_PATH=${BASE_PATH}/projecttype
+export REGION=$(grep -A1 regions: .taskcat.yml | awk '/ - / {print $NF}' |sort | uniq -c |sort -k1| head -1 |awk '{print $NF}')
 
 cd ${PROJECT_PATH}
 
@@ -15,7 +16,7 @@ cleanup_region() {
 }
 
 cleanup_all_regions() {
-    export AWS_DEFAULT_REGION=us-east-1
+    export AWS_DEFAULT_REGION=$REGION
     regions=($(aws ec2 describe-regions --query "Regions[*].RegionName" --output text))
     for region in ${regions[@]}
     do
@@ -24,13 +25,18 @@ cleanup_all_regions() {
 }
 
 run_test() {
-    echo "Running e2e test: $1"
     cleanup_all_regions
-    echo $AWS_DEFAULT_REGION
     unset AWS_DEFAULT_REGION
-    echo $AWS_DEFAULT_REGION
-    taskcat test run -t $1
-}
+    if [ -z "$1" ]; then
+        echo "Running e2e test: ALL"
+        taskcat test run -n
+        .project_automation/functional_tests/scoutsuite/scoutsuite.sh
+    else
+        echo "Running e2e test: $1"
+        taskcat test run -n -t $1
+        .project_automation/functional_tests/scoutsuite/scoutsuite.sh
+    fi
+    }
 
 # Run taskcat e2e test
 run_test "launch-qradar-main-default"

diff --git a/.project_automation/functional_tests/scoutsuite/scoutsuite.sh b/.project_automation/functional_tests/scoutsuite/scoutsuite.sh
@@ -19,7 +19,7 @@ run_scoutsuite() {
     # Upload Scoutsuite security scan results to S3 bucket named scoutsuite-results-aws-AWS-ACCOUNT-ID
     python3 .project_automation/functional_tests/scoutsuite/process-scoutsuite-report.py
     # Delete taskcat e2e test resources
-    taskcat test clean ALL
+    taskcat test clean ALL -w -r $REGION
     process_scoutsuite_report
 }
 
@@ -29,7 +29,8 @@ process_scoutsuite_report() {
     scoutsuite_s3_filename=$(cat scoutsuite_s3_filename.txt)
     rm scoutsuite_sysout.txt
     rm scoutsuite_s3_filename.txt
-    if [ "$scoutsuite_sysout_result" -ne 0 ]; then       
+    if [ "$scoutsuite_sysout_result" -ne 0 ]; 
+    then
         # The value is non-zero, indicating Scoutsuite report needs to be checked for security issues
         echo "Scoutsuite report contains security issues. For details please check the log messages above or the file $scoutsuite_s3_filename in the S3 bucket named scoutsuite-results-aws-$AWS_ACCOUNT_ID in the AWS test account provided by the ABI team."
         exit 1 

diff --git a/.project_metadata.yml b/.project_metadata.yml
@@ -1,23 +1,23 @@
-project_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln8885t0'
+project_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln8885t0'
 project_name: 'ABI IBM Security QRadar'
 project_code: '84bee679-c25e-49cd-8172-c727cc4f8fe1'
 project_type: 'ABI CloudFormation Project Type'
 partner_name: 'IBM'
 launch_date: '2023-10-04'
 repo_name: 'cfn-abi-ibmsecurity-qradar'
 templates_info:
-  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rqsi'
+  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rqsi'
     file_name: 'templates/abi-enable-qradar-integration.yaml'
     count_flag: 'true'
-  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88tcpg'
+  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88tcpg'
     file_name: 'templates/enable-integrations/enable-cloudtrail-integrations.yaml'
     count_flag: 'false'
-  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rrkg'
+  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rrkg'
     file_name: 'templates/enable-integrations/enable-guardduty-integrations.yaml'
     count_flag: 'false'
-  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rt4b'
+  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rt4b'
     file_name: 'enable-integrations/enable-sqs-s3-integrations.yaml'
     count_flag: 'false'
-  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rscd'
+  - template_id: 'abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rscd'
     file_name: 'enable-integrations/setup-iam-role.yaml'
     count_flag: 'false'
diff --git a/scripts/cleanup_config.json b/scripts/cleanup_config.json
@@ -1,4 +1,8 @@
 [
+  {
+    "Type" : "STACK",
+    "Filter" : "tCaT-launch-qradar"
+  },
   {
     "Type" : "SSM_PARAMETER",
     "Filter" : "/sra/gd/",
@@ -88,6 +92,14 @@
     "Type" : "CODE_BUILD",
     "Filter": "sra-codebuild-project"
   },
+  {
+    "Type" : "STACK_SET",
+    "Filter" : "sra-stackset-execution-role"
+  },
+  {
+    "Type" : "STACK",
+    "Filter" : "sra-common-prerequisites-staging-s3-bucket"
+  },
   {
     "Type" : "IAM_ROLE",
     "Filter" : "sra-execution",
@@ -103,18 +115,7 @@
     "Account" : "audit"
   },
   {
-    "Type" : "STACK_SET",
-    "Filter" : "sra-stackset-execution-role"
-  },
-  {
-    "Type" : "STACK",
-    "Filter" : "sra-common-prerequisites-staging-s3-bucket"
-  },
-  {
-    "Type" : "STACK",
-    "Filter" : "tCaT-launch-qradar"
-  },
-  {
-    "Type" : "GUARDDUTY_DET"
+      "Type" : "STACK",
+      "Filter" : "Lambda-S3-PresignedURL"
   }
 ]
diff --git a/scripts/cleanup_config.py b/scripts/cleanup_config.py
@@ -23,7 +23,13 @@
 GD  = SESSION.client('guardduty')
 
 STACKSTATUS = [ 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE', 'DELETE_FAILED', 'DELETE_COMPLETE']
-
+VALID_STATUS = ['CREATE_IN_PROGRESS', 'CREATE_FAILED', 'CREATE_COMPLETE',
+                'ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE',
+                'DELETE_IN_PROGRESS', 'DELETE_FAILED',
+                'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS',
+                'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_IN_PROGRESS',
+                'UPDATE_ROLLBACK_FAILED', 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
+                'UPDATE_ROLLBACK_COMPLETE', 'REVIEW_IN_PROGRESS']
 def list_stacksets():
     '''List all stacksets in the account'''
     response = CF.list_stack_sets()
@@ -145,11 +151,11 @@ def delete_stacksets(filters):
         CF.delete_stack_set(StackSetName=cf_name)
 
 def list_all_stacks():
-    '''List all stacks in the account'''
-    response = CF.list_stacks()
+    '''List all stacks in the account with status other than DELETE_COMPLETE'''
+    response = CF.list_stacks(StackStatusFilter=VALID_STATUS)
     stacks = response['StackSummaries']
     while response.get('NextToken'):
-        response = CF.list_stacks(NextToken=response['NextToken'])
+        response = CF.list_stacks(StackStatusFilter=VALID_STATUS, NextToken=response['NextToken'])
         stacks.extend(response['StackSummaries'])
     return stacks
 
@@ -170,20 +176,31 @@ def is_nested_stack(stack_name):
         result = True
     return result
 
+def list_stacks_by_prefix(stack_prefix):
+    '''List stacks by prefix'''
+    stacks = list_all_stacks()
+    output = []
+    for stack in stacks:
+        if stack['StackName'].startswith(stack_prefix):
+            output.append(stack['StackName'])
+    return sorted(output, key=len)
+
 def delete_stack(filters='tCaT-'):
     '''Delete all stacks created by CfCT solution in the account'''
-    stacks = list_all_stacks()
+    stacks = list_stacks_by_prefix(filters)
     for stack in stacks:
-        stack_name = stack['StackName']
-        stack_status = stack['StackStatus']
-        if stack_name.startswith(filters) and stack_status != 'DELETE_COMPLETE':
-            print('Deleting stack: %s', stack_name)
-            CF.delete_stack(StackName=stack_name)
+        status = list_stack_status_by_name(stack)
+        if status:
+            print(f"Deleting stack: {stack}")
+            CF.delete_stack(StackName=stack)
             wait = 1
-            while list_stack_status_by_name(stack_name) not in STACKSTATUS and wait < 60:
-                print('Wait: %s, Stack: %s', stack_name, wait)
-                sleep(10)
+            stack_status = list_stack_status_by_name(stack)
+            while stack_status and stack_status not in STACKSTATUS and wait < 60:
+                sleep_time = 15-wait/6
+                print(f"Wait: {stack}, {wait}, {sleep_time}, {stack_status}")
+                sleep(sleep_time)
                 wait += 1
+                stack_status = list_stack_status_by_name(stack)
 
 def delete_all_objects_from_s3_bucket(bucket_name, account=None):
     '''Delete all objects from an S3 bucket'''
@@ -228,14 +245,15 @@ def delete_s3_buckets(item):
                 else:
                     raise exe
 
-def list_all_parameters(ssm_session=SSM):
+def list_all_parameters(ssm_session):
     ''''List all parameters in the account'''
     response = ssm_session.describe_parameters()
-    parameters = response['Parameters']
+    result = response['Parameters']
     while response.get('NextToken'):
         response = ssm_session.describe_parameters(NextToken=response['NextToken'])
-        parameters.extend(response['Parameters'])
-    return parameters
+        result.extend(response['Parameters'])
+
+    return result
 
 def delete_parameters(item):
     '''Delete all parameters created in the account'''
@@ -390,6 +408,7 @@ def delete_detector():
                 print('Deleting GuardDuty Detector in %s', account['Id'])
                 gd_client.delete_detector(DetectorId=det_id)
 
+
 def list_cb_projects():
     '''
     List all CodeBuild projects

diff --git a/templates/abi-enable-qradar-integration.yaml b/templates/abi-enable-qradar-integration.yaml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: "QRadar Integration (abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rqsi)"
+Description: "QRadar Integration (abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rqsi)"
 
 Parameters:
   PrincipalArn:

diff --git a/templates/enable-integrations/enable-cloudtrail-integrations.yaml b/templates/enable-integrations/enable-cloudtrail-integrations.yaml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: "CloudTrail Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88tcpg)"
+Description: "CloudTrail Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88tcpg)"
 
 Parameters:
   pSRASourceS3BucketName:

diff --git a/templates/enable-integrations/enable-guardduty-integrations.yaml b/templates/enable-integrations/enable-guardduty-integrations.yaml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: "GuardDuty Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rrkg)"
+Description: "GuardDuty Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rrkg)"
 
 Parameters:
   pSRASourceS3BucketName:

diff --git a/templates/enable-integrations/enable-sqs-s3-integrations.yaml b/templates/enable-integrations/enable-sqs-s3-integrations.yaml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: "SQS and S3 Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rt4b)"
+Description: "SQS and S3 Integration for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rt4b)"
 Parameters:
   pBucketName:
     Type: String

diff --git a/templates/enable-integrations/setup-iam-role.yaml b/templates/enable-integrations/setup-iam-role.yaml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: "Setup IAM role for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-5Qgs4qOsPgbXnTtlFjeTTo-ln88rscd)"
+Description: "Setup IAM role for QRadar. (abp-1kirFQBF75MfEQ3RbMQHRb-42uebFeAkVOI7f1QU0xH7b-ln88rscd)"
 Parameters:
   PrincipalArn:
     Type: String