From d6ee7d3b11fd407ea7c64c42ef62691c2fe91ec9 Mon Sep 17 00:00:00 2001 From: Idan Koren Date: Tue, 25 Apr 2023 16:50:49 +0300 Subject: [PATCH 1/3] add_rds_count --- .gitignore | 2 ++ aws_resource_count.py | 43 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac340b5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.log +*.idea diff --git a/aws_resource_count.py b/aws_resource_count.py index a25c981..fcc8694 100644 --- a/aws_resource_count.py +++ b/aws_resource_count.py @@ -181,36 +181,60 @@ def get_region_cluster_nodes(session: CoveSession, service_name: str, region_nam return count +@retry +def get_region_managed_dbs(session: CoveSession, service_name: str, region_name: Optional[str] = None) -> int: + if hasattr(session, "session_information"): + region_name = session.session_information['Region'] + client = session.client("rds", region_name=region_name) + paginator = client.get_paginator("describe_db_instances") + count = 0 + clusters_set = set() + for page in paginator.paginate(): + for db_instance in page["DBInstances"]: + if not db_instance.get('ReadReplicaSourceDBInstanceIdentifier'): + if cluster_name := db_instance.get('DBClusterIdentifier'): + clusters_set.add(cluster_name) + else: + count += 1 + count += len(clusters_set) + return count + + SERVICES_CONF: Dict[str, Any] = { "ec2": { "function": get_region_instances, "display_name": "Virtual Machines", - "workload_units": 1 + "workload_units_ratio": 1 }, "lambda": { "function": get_region_functions, "display_name": "Serverless Functions", - "workload_units": 50 + "workload_units_ratio": 1 / 50 }, "ecr": { "function": get_region_ecr_repos, "display_name": "Container Images", - "workload_units": 10 + "workload_units_ratio": 1 / 10 }, "ami": { "function": get_region_vm_images, "display_name": "VM Images", - "workload_units": 1 + "workload_units_ratio": 1 }, "ecs": { "function": get_region_serverless_containers, "display_name": "Serverless Containers", - "workload_units": 10 + "workload_units_ratio": 1 / 10 }, "eks": { "function": get_region_cluster_nodes, "display_name": "Container Hosts", - "workload_units": 1 + "workload_units_ratio": 1 + }, + "rds": { + "function": get_region_managed_dbs, + "display_name": "Managed Data Stores", + "workload_units_ratio": 2 } } @@ -235,14 +259,14 @@ def current_account_resources_count(session: boto3.Session) -> Dict[str, int]: return total_results -def print_results(results: Dict[str, int], account_id: Optional[str]=None) -> None: +def print_results(results: Dict[str, int], account_id: Optional[str] = None) -> None: log_total_results = account_id is None result_str = "\n==============\nTotal results:\n==============\n" if log_total_results else f"AWS Account number: [{account_id}]\n" total_workloads = 0 for service, count in results.items(): if service == "ecr": count = count * 1.1 # we scan 2 images per one repository and we decided to multiply the count by 1.1 based on production statistics - workloads = round(count / SERVICES_CONF[service]['workload_units']) + workloads = round(count * SERVICES_CONF[service]['workload_units_ratio']) if workloads == 0 and count > 0: workloads = 1 result_str += f"{SERVICES_CONF[service]['display_name']} Count: {round(count)}{f' (Workload Units: {workloads})' if log_total_results else ''}\n" @@ -309,6 +333,9 @@ def main(): _parser.add_argument("--skip-container-hosts", action="store_true", help=f"Skip counting {SERVICES_CONF['eks']['display_name']}") + _parser.add_argument("--skip-managed-data-sources", action="store_true", + help=f"Skip counting {SERVICES_CONF['rds']['display_name']}") + _parser.add_argument("--show-logs-per-account", action="store_true", help=f"Log resource count per AWS account") From 42f70a3e473cf35e748ebca3d3024697038300b1 Mon Sep 17 00:00:00 2001 From: Idan Koren Date: Thu, 27 Apr 2023 12:35:14 +0300 Subject: [PATCH 2/3] add_azure_sql_dbs --- azure_resource_count.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/azure_resource_count.sh b/azure_resource_count.sh index f25aedd..c41223a 100644 --- a/azure_resource_count.sh +++ b/azure_resource_count.sh @@ -2,11 +2,12 @@ LOG_FILE='azure_resource_count.log' WORKLOAD_VM_UNITS=1 -WORKLOAD_FUNCTION_UNITS=50 -WORKLOAD_SERVERLESS_CONTAINER_UNITS=10 +WORKLOAD_FUNCTION_UNITS=50 # (1 / 50) +WORKLOAD_SERVERLESS_CONTAINER_UNITS=10 # (1 / 10) WORKLOAD_VM_IMAGE_UNITS=1 -WORKLOAD_CONTAINER_IMAGE_UNITS=10 +WORKLOAD_CONTAINER_IMAGE_UNITS=10 # (1 / 10) WORKLOAD_CONTAINER_HOST_UNITS=1 +WORKLOAD_DATABASE_UNITS=2 _tmp_files=$(mktemp) cleanup() { @@ -85,6 +86,7 @@ ContainerCount=0 containerImageCount=0 vmImageCount=0 aksNodesCount=0 +SqlDbCount=0 # Set a counter for progress indicator counter=0 @@ -191,6 +193,21 @@ for subscription in $subscriptions; do aksNodesCount=$((aksNodesCount + currentNodesCount)) fi + # Get the number of Azure SQL Databases + az sql server list --subscription $subscription --query "[].{name: name, resourceGroup: resourceGroup}" -o json> ${_temp_subscription_output} 2>> $LOG_FILE || echo "Failed to get Managed Data Stores for subscription ${subscription}" + servers=$(cat "${_temp_subscription_output}") + currentNodesCount=0 + for cluster in $(echo "$servers" | jq -c '.[]'); do + server_name=$(echo $cluster | jq -r '.name') + rg_name=$(echo $cluster | jq -r '.resourceGroup') + az sql db list --subscription $subscription --resource-group $rg_name --server $server_name --query "[?name!='master'] | length(@)" -o tsv > ${_temp_subscription_output} 2>> $LOG_FILE || echo "Failed to get Managed Data Stores for subscription ${subscription}" + currentDBCount=$(cat "${_temp_subscription_output}") + if [ -n "$currentDBCount" ]; then + SqlDbCount=$((SqlDbCount + currentDBCount)) + echo "Managed Data Stores Count: $currentDBCount" + fi + done + #Increment counter counter=$((counter+1)) if [ -n "$management_group" ]; then @@ -227,7 +244,11 @@ container_host_workloads=$(( ( aksNodesCount + WORKLOAD_CONTAINER_HOST_UNITS / 2 if [[ $container_host_workloads -eq 0 && $aksNodesCount -gt 0 ]]; then container_host_workloads=1 fi -total_workloads=$(( vm_workloads + function_workloads + container_workloads + container_image_workloads + vm_image_workloads + container_host_workloads )) +databases_workloads=$(( SqlDbCount * WORKLOAD_DATABASE_UNITS )) +if [[ $databases_workloads -eq 0 && $SqlDbCount -gt 0 ]]; then + databases_workloads=1 +fi +total_workloads=$(( vm_workloads + function_workloads + container_workloads + container_image_workloads + vm_image_workloads + container_host_workloads + databases_workloads)) echo "==============" echo "Total results:" @@ -238,6 +259,7 @@ echo "Serverless Containers Count: $ContainerCount (Workload Units: ${container_ echo "Container Images Count: $containerImageCount (Workload Units: ${container_image_workloads})" echo "VM Images Count: $vmImageCount (Workload Units: ${vm_image_workloads})" echo "Container Hosts Count: $aksNodesCount (Workload Units: ${container_host_workloads})" +echo "Managed Data Stores Count: $SqlDbCount (Workload Units: ${databases_workloads})" echo "--------------------------------------" echo "TOTAL Estimated Workload Units: ${total_workloads}" echo From 73ed6402ff5030c390f0ee4239ddc8ad421d1bc6 Mon Sep 17 00:00:00 2001 From: Idan Koren Date: Thu, 27 Apr 2023 19:25:15 +0300 Subject: [PATCH 3/3] add_gcp_sql_instances --- azure_resource_count.sh | 19 ++++++++++--------- gcp_resource_count.sh | 22 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 13 deletions(-) mode change 100644 => 100755 azure_resource_count.sh diff --git a/azure_resource_count.sh b/azure_resource_count.sh old mode 100644 new mode 100755 index c41223a..01fad55 --- a/azure_resource_count.sh +++ b/azure_resource_count.sh @@ -196,17 +196,18 @@ for subscription in $subscriptions; do # Get the number of Azure SQL Databases az sql server list --subscription $subscription --query "[].{name: name, resourceGroup: resourceGroup}" -o json> ${_temp_subscription_output} 2>> $LOG_FILE || echo "Failed to get Managed Data Stores for subscription ${subscription}" servers=$(cat "${_temp_subscription_output}") - currentNodesCount=0 - for cluster in $(echo "$servers" | jq -c '.[]'); do - server_name=$(echo $cluster | jq -r '.name') - rg_name=$(echo $cluster | jq -r '.resourceGroup') + currentDbInSub=0 + for server in $(echo "$servers" | jq -c '.[]'); do + server_name=$(echo $server | jq -r '.name') + rg_name=$(echo $server | jq -r '.resourceGroup') az sql db list --subscription $subscription --resource-group $rg_name --server $server_name --query "[?name!='master'] | length(@)" -o tsv > ${_temp_subscription_output} 2>> $LOG_FILE || echo "Failed to get Managed Data Stores for subscription ${subscription}" - currentDBCount=$(cat "${_temp_subscription_output}") - if [ -n "$currentDBCount" ]; then - SqlDbCount=$((SqlDbCount + currentDBCount)) - echo "Managed Data Stores Count: $currentDBCount" - fi + currentDbsPerServerCount=$(cat "${_temp_subscription_output}") + currentDbInSub=$((currentDbInSub + currentDbsPerServerCount)) done + if [ -n "$currentDbInSub" ]; then + SqlDbCount=$((SqlDbCount + currentDbInSub)) + echo "Managed Data Stores Count: $currentDbInSub" + fi #Increment counter counter=$((counter+1)) diff --git a/gcp_resource_count.sh b/gcp_resource_count.sh index 4052ba6..ea620a4 100644 --- a/gcp_resource_count.sh +++ b/gcp_resource_count.sh @@ -3,11 +3,12 @@ set -e LOG_FILE='gcp_resource_count.log' WORKLOAD_VM_UNITS=1 -WORKLOAD_FUNCTION_UNITS=50 -WORKLOAD_SERVERLESS_CONTAINER_UNITS=10 +WORKLOAD_FUNCTION_UNITS=50 # (1 / 50) +WORKLOAD_SERVERLESS_CONTAINER_UNITS=10 # (1 / 10) WORKLOAD_VM_IMAGE_UNITS=1 -WORKLOAD_CONTAINER_IMAGE_UNITS=10 +WORKLOAD_CONTAINER_IMAGE_UNITS=10 # (1 / 10) WORKLOAD_CONTAINER_HOST_UNITS=1 +WORKLOAD_DATABASE_UNITS=2 _tmp_files=$(mktemp) @@ -59,6 +60,7 @@ total_cloud_run=0 total_container_images=0 total_vm_images=0 total_gke_nodes=0 +total_dbs=0 counter=0 _temp_project_output=$(_make_temp_file) for project in $PROJECTS; do @@ -122,6 +124,13 @@ for project in $PROJECTS; do echo "Container Hosts Count: $project_nodes_count" fi + gcloud -q sql instances list --project "${project}" --filter='-masterInstanceName:* AND settings.activationPolicy!='NEVER' AND settings.backupConfiguration.enabled=true' --format=json > ${_temp_project_output} 2>> $LOG_FILE || echo "Failed to get Managed Data Stores for project ${project}" + project_dbs_count=$(cat "${_temp_project_output}" | jq -r '. | length') + if [ -n "$project_dbs_count" ]; then + total_dbs=$((total_dbs + project_dbs_count)) + echo "Managed Data Stores Count: $project_dbs_count" + fi + #Increment counter counter=$((counter+1)) if [ -n "$PROJECT_LEN" ]; then @@ -159,7 +168,11 @@ container_host_workloads=$(( ( total_gke_nodes + WORKLOAD_CONTAINER_HOST_UNITS / if [[ $container_host_workloads -eq 0 && $total_gke_nodes -gt 0 ]]; then container_host_workloads=1 fi -total_workloads=$(( vm_workloads + function_workloads + container_workloads + container_image_workloads + vm_image_workloads + container_host_workloads )) +db_workloads=$(( total_dbs * WORKLOAD_DATABASE_UNITS )) +if [[ $db_workloads -eq 0 && $total_dbs -gt 0 ]]; then + db_workloads=1 +fi +total_workloads=$(( vm_workloads + function_workloads + container_workloads + container_image_workloads + vm_image_workloads + container_host_workloads + db_workloads)) echo "==============" echo "Total results:" @@ -170,6 +183,7 @@ echo "Serverless Containers Count: $total_cloud_run (Workload Units: ${container echo "Container Images Count: $total_container_images (Workload Units: ${container_image_workloads})" echo "VM Images Count: $total_vm_images (Workload Units: ${vm_image_workloads})" echo "Container Hosts Count: $total_gke_nodes (Workload Units: ${container_host_workloads})" +echo "Managed Data Stores Count: $total_dbs (Workload Units: ${db_workloads})" echo "--------------------------------------" echo "TOTAL Estimated Workload Units: ${total_workloads}" echo