Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Count supported managed DB resources. #9

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Count supported managed DB resources. #9

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0058046
Count supported managed DB resources.
kanstantsink-orca Dec 10, 2024
7e53312
Address comments
kanstantsink-orca Dec 11, 2024
3d5d09c
Update aws_resource_count.py
kanstantsink-orca Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 82 additions & 1 deletion aws_resource_count.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,54 @@ def get_region_cluster_nodes(session: CoveSession, service_name: str, region_nam
return count



@retry
def get_region_rds_resources(session: CoveSession, service_name: str, region_name: Optional[str] = None) -> int:
supported_db_engines = ["postgres", "mysql", "mongodb"]
if hasattr(session, "session_information"):
region_name = session.session_information['Region']
client = session.client("rds", region_name=region_name)
paginator = client.get_paginator("describe_db_instances")
count = 0
clusters_set = set()
for page in paginator.paginate():
for db_instance in page["DBInstances"]:
db_engine = db_instance["Engine"]
db_engine_normalized = db_engine.replace("aurora-", "").replace("postgresql", "postgres")
if db_engine_normalized in supported_db_engines:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we consider size too? i.e. > 1tb

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it should be an option to the runner script weather to count all or under 1TB only

if not db_instance.get('ReadReplicaSourceDBInstanceIdentifier'):
if cluster_name := db_instance.get('DBClusterIdentifier'):
clusters_set.add(cluster_name)
else:
count += 1
count += len(clusters_set)
return count


@retry
def get_region_dynamodb_resources(session: CoveSession, service_name: str, region_name: Optional[str] = None) -> int:
if hasattr(session, "session_information"):
region_name = session.session_information['Region']
client = session.client("dynamodb", region_name=region_name)
paginator = client.get_paginator("list_tables")
count = 0
for page in paginator.paginate():
count += len(page["TableNames"])
return count


@retry
def get_region_redshift_resources(session: CoveSession, service_name: str, region_name: Optional[str] = None) -> int:
if hasattr(session, "session_information"):
region_name = session.session_information['Region']
client = session.client("redshift", region_name=region_name)
paginator = client.get_paginator("describe_clusters")
count = 0
for page in paginator.paginate():
count += len(page["Clusters"])
return count


SERVICES_CONF: Dict[str, Any] = {
"ec2": {
"function": get_region_instances,
Expand Down Expand Up @@ -211,7 +259,22 @@ def get_region_cluster_nodes(session: CoveSession, service_name: str, region_nam
"function": get_region_cluster_nodes,
"display_name": "Container Hosts",
"workload_units": 1
}
},
"rds": {
"function": get_region_rds_resources,
"display_name": "RDS Instances and Clusters",
"workload_units": 1
},
"dynamodb": {
"function": get_region_dynamodb_resources,
"display_name": "DynamoDB Tables",
Copy link
Collaborator

@amine-orca amine-orca Dec 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we expect same display names across providers, so we should have Managed Databases and DataWarehouses

"workload_units": 50
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where does this number come from?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we still need to confirm it. For now I put it same as Lambda functions

},
"redshift": {
"function": get_region_redshift_resources,
"display_name": "Redshift Clusters",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We may want to merge the count of DDB + RedShift cause I think we want one value for DataWarehouses

"workload_units": 50
},
}

ALL_REGIONS = [r["RegionName"] for r in boto3.client("ec2").describe_regions()["Regions"]]
Expand Down Expand Up @@ -281,6 +344,15 @@ def set_skip_resources(args: argparse.Namespace) -> None:
if args.skip_container_hosts:
skipped_resources.append(SERVICES_CONF["eks"]['display_name'])
SERVICES_CONF.pop("eks")
if args.skip_rds:
skipped_resources.append(SERVICES_CONF["rds"]['display_name'])
SERVICES_CONF.pop("rds")
if args.skip_dynamodb:
skipped_resources.append(SERVICES_CONF["dynamodb"]['display_name'])
SERVICES_CONF.pop("dynamodb")
if args.skip_redshift:
skipped_resources.append(SERVICES_CONF["redshift"]['display_name'])
SERVICES_CONF.pop("redshift")
if skipped_resources:
logger.info(f"Skip counting the following resources: {', '.join(skipped_resources)}.")

Expand Down Expand Up @@ -311,6 +383,15 @@ def main():
_parser.add_argument("--skip-container-hosts", action="store_true",
help=f"Skip counting {SERVICES_CONF['eks']['display_name']}")

_parser.add_argument("--skip-rds", action="store_true",
help=f"Skip counting {SERVICES_CONF['rds']['display_name']}")

_parser.add_argument("--skip-dynamodb", action="store_true",
help=f"Skip counting {SERVICES_CONF['dynamodb']['display_name']}")

_parser.add_argument("--skip-redshift", action="store_true",
help=f"Skip counting {SERVICES_CONF['redshift']['display_name']}")

_parser.add_argument("--show-logs-per-account", action="store_true",
help=f"Log resource count per AWS account")

Expand Down