Merge pull request #59 from salesforce/refactor/naming-and-structure

Improve naming and structure

azure_guardrails/command/create_parameters_file.py

@@ -6,12 +6,12 @@
 import logging
 from pathlib import Path
 import click
-from click_option_group import optgroup, RequiredMutuallyExclusiveOptionGroup
+from click_option_group import optgroup
 from azure_guardrails import set_log_level
 # from azure_guardrails.shared.parameters_categorized import get_parameters_template
-from azure_guardrails.shared.parameters_template import ParameterTemplate
-from azure_guardrails.shared import utils, validate
-from azure_guardrails.shared.config import Config, get_default_config, get_config_from_file
+from azure_guardrails.templates.parameters_template import ParameterTemplate
+from azure_guardrails.shared import validate
+from azure_guardrails.shared.config import get_default_config, get_config_from_file
 
 logger = logging.getLogger(__name__)
 

azure_guardrails/command/describe_policy.py

@@ -2,13 +2,12 @@
 Supply a Policy's display name or a policy ID and get some metadata about the policy.
 """
 import logging
-import yaml
 import json
 import ruamel.yaml
 import click
 from azure_guardrails import set_log_level
 from click_option_group import optgroup, RequiredMutuallyExclusiveOptionGroup
-from azure_guardrails.shared.iam_definition import AzurePolicies
+from azure_guardrails.iam_definition.azure_policies import AzurePolicies
 from azure_guardrails.shared.config import get_empty_config
 logger = logging.getLogger(__name__)
 

azure_guardrails/command/generate_terraform.py

@@ -8,10 +8,10 @@
 from azure_guardrails import set_log_level
 from azure_guardrails.terraform.terraform_no_params import TerraformTemplateNoParams
 from azure_guardrails.terraform.terraform_with_params import TerraformTemplateWithParamsV5
-from azure_guardrails.shared.iam_definition import AzurePolicies
+from azure_guardrails.iam_definition.azure_policies import AzurePolicies
 from azure_guardrails.shared import utils, validate
 from azure_guardrails.shared.config import get_default_config, get_config_from_file
-from azure_guardrails.shared.parameters_categorized import OverallCategorizedParameters
+from azure_guardrails.shared.parameters_categorized import CategorizedParameters
 
 logger = logging.getLogger(__name__)
 
@@ -192,7 +192,7 @@ def generate_terraform(
             parameters_config = utils.read_yaml_file(parameters_config_file)
         else:
             parameters_config = None
-        categorized_parameters = OverallCategorizedParameters(
+        categorized_parameters = CategorizedParameters(
             azure_policies=azure_policies,
             parameters_config=parameters_config,
             params_required=params_required,

azure_guardrails/command/list_policies.py

@@ -1,14 +1,12 @@
 """
 
 """
-import os
 import logging
-import json
 import yaml
 import click
 from click_option_group import optgroup, RequiredMutuallyExclusiveOptionGroup
 from azure_guardrails import set_log_level
-from azure_guardrails.shared.iam_definition import AzurePolicies
+from azure_guardrails.iam_definition.azure_policies import AzurePolicies
 from azure_guardrails.shared import utils, validate
 
 logger = logging.getLogger(__name__)

azure_guardrails/shared/iam_definition.py → azure_guardrails/iam_definition/azure_policies.py

@@ -10,7 +10,7 @@
 import logging
 from azure_guardrails.shared import utils
 from azure_guardrails.shared.config import DEFAULT_CONFIG, Config
-from azure_guardrails.guardrails.policy_definition import PolicyDefinition
+from azure_guardrails.iam_definition.policy_definition import PolicyDefinition
 
 logger = logging.getLogger(__name__)
 
@@ -338,7 +338,7 @@ def compliance_coverage_data(self, no_params: bool = True, params_optional: bool
         results = {}
         compliance_data_file = os.path.abspath(
             os.path.join(
-                os.path.dirname(__file__), os.path.pardir, "shared", "data", "compliance-data.json"
+                os.path.dirname(__file__), os.path.pardir, "shared", "../shared/data", "compliance-data.json"
             )
         )
         with open(compliance_data_file) as json_file:

azure_guardrails/guardrails/policy_definition.py → azure_guardrails/iam_definition/policy_definition.py

@@ -1,6 +1,6 @@
 import logging
 import json
-from azure_guardrails.guardrails.properties import Properties
+from azure_guardrails.iam_definition.properties import Properties
 from azure_guardrails.shared import utils
 logger = logging.getLogger(__name__)
 

azure_guardrails/guardrails/properties.py → azure_guardrails/iam_definition/properties.py

@@ -1,5 +1,5 @@
 import json
-from azure_guardrails.guardrails.parameter import Parameter
+from azure_guardrails.iam_definition.parameter import Parameter
 
 
 class Properties:

azure_guardrails/scrapers/parse_builtin_definitions.py

@@ -2,7 +2,7 @@
 import json
 import copy
 from azure_guardrails.shared import utils
-from azure_guardrails.guardrails.policy_definition import PolicyDefinition
+from azure_guardrails.iam_definition.policy_definition import PolicyDefinition
 default_service_names = utils.get_service_names()
 default_service_names.sort()
 

azure_guardrails/shared/config.py

@@ -1,31 +1,11 @@
-import os
 import logging
 import json
 import yaml
-from jinja2 import Template, Environment, FileSystemLoader
 from azure_guardrails.shared import utils
+from azure_guardrails.templates.config_template import get_config_template
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_CONFIG_FILE = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "config-template.yml")
-)
-
-
-def get_config_template() -> str:
-    template_contents = dict(
-        match_only_keywords=[],
-        exclude_keywords=[],
-        service_names=utils.get_service_names(),
-    )
-    template_path = os.path.join(os.path.dirname(__file__))
-    env = Environment(loader=FileSystemLoader(template_path))  # nosec
-    template = env.get_template("config-template.yml")
-    return template.render(t=template_contents)
-
-
-DEFAULT_CONFIG_TEMPLATE = get_config_template()
-
 
 class Config:
     def __init__(
@@ -208,9 +188,6 @@ def get_default_config(exclude_services: list = None, match_only_keywords: list
     return config
 
 
-DEFAULT_CONFIG = get_default_config()
-
-
 def get_config_from_file(config_file: str, exclude_services: list = None) -> Config:
     with open(config_file, "r") as yaml_file:
         config_cfg = yaml.safe_load(yaml_file)
@@ -246,3 +223,7 @@ def get_empty_config() -> Config:
         exclude_keywords=None
     )
     return config
+
+
+DEFAULT_CONFIG_TEMPLATE = get_config_template()
+DEFAULT_CONFIG = get_default_config()

azure_guardrails/shared/parameters_categorized.py

@@ -1,42 +1,13 @@
-from jinja2 import Template, Environment, FileSystemLoader
-import os
-import json
-import copy
-import yaml
 import logging
-from azure_guardrails.shared.iam_definition import AzurePolicies
+from azure_guardrails.iam_definition.azure_policies import AzurePolicies
 from azure_guardrails.shared.config import DEFAULT_CONFIG
 from azure_guardrails.shared import utils
-from azure_guardrails.guardrails.policy_definition import PolicyDefinition
+from azure_guardrails.iam_definition.policy_definition import PolicyDefinition
 
 logger = logging.getLogger(__name__)
 
 
-def get_parameters_template() -> str:
-    # if not categorized_parameters:
-    #     with open(DEFAULT_PARAMETERS_FILE, "r") as file:
-    #         default_categorized_parameters = yaml.safe_load(file)
-    #     categorized_parameters = default_categorized_parameters
-    azure_policies = AzurePolicies(service_names=["all"], config=DEFAULT_CONFIG)
-    categorized_parameters = OverallCategorizedParameters(azure_policies=azure_policies, params_optional=True, params_required=True, audit_only=False)
-    template_contents = dict(
-        categorized_parameters=categorized_parameters.service_categorized_parameters,
-    )
-    template_path = os.path.join(os.path.dirname(__file__))
-    env = Environment(loader=FileSystemLoader(template_path))  # nosec
-
-    def is_list(value):
-        return isinstance(value, list)
-
-    env.tests['is_a_list'] = is_list
-    env.filters["debug"] = print
-    env.filters['tojson'] = json.dumps
-    env.tests['is_none_instance'] = utils.is_none_instance
-    template = env.get_template("parameters-template.yml.j2")
-    return template.render(t=template_contents)
-
-
-class OverallCategorizedParameters:
+class CategorizedParameters:
     """Feed the results of the JSON File into here and store it in the class structure"""
 
     def __init__(
@@ -134,11 +105,8 @@ def set_service_categorized_parameters(self):
                 policy_definition = self.azure_policies.get_policy_definition(policy_id=policy_details.get("short_id"))
                 # See if it has parameters
                 if not policy_definition.parameters:
-                    # if "parameters" not in policy_details.keys():
                     continue
                 results[service_name][policy_name] = {}
-                # policy_definition = self.azure_policies.get_policy_definition_by_display_name(display_name=policy_name)
-                # policy_parameters = policy_details.get("parameters", None)
 
                 for parameter_name in policy_definition.parameters:
                     if policy_definition.properties.parameters[parameter_name].allowed_values:
@@ -232,11 +200,9 @@ def get_parameter_value_from_config(self, display_name: str, parameter_name: str
                     user_supplied_value = {}
                     return user_supplied_value
                 else:
-                    # TODO: Should throw an exception here. Let the user know that they need to supply a value!
                     logger.debug(
                         f"Parameter value not supplied by user. No default value available. Parameter: {parameter_name}. Policy ID: {policy_id}")
                     return None
-            # TODO: How do we differentiate between when Azure says an empty list is okay vs when it is not?
             else:
                 if isinstance(user_supplied_value, list):
                     user_supplied_value = []
@@ -255,34 +221,3 @@ def get_parameter_value_from_config(self, display_name: str, parameter_name: str
                 f"Parameter supplied by user. Using user-supplied value. Parameter: {parameter_name}. Value: {user_supplied_value}. Policy ID: {policy_id}")
             return user_supplied_value
 
-    def parameters(self) -> dict:
-        results = {}
-        policy_id_pairs = self.azure_policies.get_all_policy_ids_sorted_by_service(
-            no_params=False, params_optional=self.params_optional, params_required=self.params_required,
-            audit_only=self.audit_only)
-        for service_name, service_policies in policy_id_pairs.items():
-            results[service_name] = {}
-            for policy_definition_name, policy_definition_details in service_policies.items():
-                results[service_name][policy_definition_name] = {}
-                if "parameters" in policy_definition_details.keys():
-                    for parameter_name, parameter_details in policy_definition_details.get("parameters").items():
-                        if isinstance(parameter_details.get("value", None), type(None)):
-                            if isinstance(parameter_details.get("default_value", None), type(None)):
-                                value = None
-                            else:
-                                value = parameter_details.get("default_value")
-                        else:
-                            value = parameter_details.get("value")
-                        parameter = dict(
-                            name=parameter_name,
-                            service=service_name,
-                            policy_definition_name=policy_definition_name,
-                            initiative_parameters_json=parameter_details,
-                            parameter_type=parameter_details.get("type"),
-                            default_value=parameter_details.get("default_value"),
-                            value=value,
-                        )
-                        results[service_name][policy_definition_name][parameter_name] = parameter
-                else:
-                    logger.warning(f"No parameters provided. Policy Name: \"{policy_definition_name}\"")
-        return results

azure_guardrails/templates/config_template.py

@@ -0,0 +1,19 @@
+import os
+from azure_guardrails.shared import utils
+from jinja2 import Template, Environment, FileSystemLoader
+
+DEFAULT_CONFIG_FILE = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "config-template.yml.j2")
+)
+
+
+def get_config_template() -> str:
+    template_contents = dict(
+        match_only_keywords=[],
+        exclude_keywords=[],
+        service_names=utils.get_service_names(),
+    )
+    template_path = os.path.join(os.path.dirname(__file__))
+    env = Environment(loader=FileSystemLoader(template_path))  # nosec
+    template = env.get_template("config-template.yml.j2")
+    return template.render(t=template_contents)

azure_guardrails/shared/parameters_template.py → azure_guardrails/templates/parameters_template.py

@@ -1,10 +1,10 @@
-from azure_guardrails.shared.iam_definition import AzurePolicies
-from azure_guardrails.shared.parameters_categorized import OverallCategorizedParameters
-from jinja2 import Template, Environment, FileSystemLoader
+import os
+import json
+from jinja2 import Environment, FileSystemLoader
+from azure_guardrails.iam_definition.azure_policies import AzurePolicies
+from azure_guardrails.shared.parameters_categorized import CategorizedParameters
 from azure_guardrails.shared.config import DEFAULT_CONFIG, Config
 from azure_guardrails.shared import utils
-from azure_guardrails.guardrails.policy_definition import PolicyDefinition
-import os
 
 
 class ParameterSegment:
@@ -16,6 +16,18 @@ def __init__(self, parameter_name: str, parameter_type: str, value=None, default
         self.default_value = default_value
         self.value = value
 
+    def json(self):
+        return dict(
+            name=self.name,
+            type=self.type,
+            allowed_values=self.allowed_values,
+            default_value=self.default_value,
+            value=self.value
+        )
+
+    def __repr__(self) -> str:
+        return json.dumps(self.__dict__)
+
 
 class ParameterTemplate:
     def __init__(
@@ -25,15 +37,29 @@ def __init__(
         params_required: bool = False,
     ):
         self.azure_policies = AzurePolicies(service_names=["all"], config=config)
-        categorized_parameters = OverallCategorizedParameters(
+        categorized_parameters = CategorizedParameters(
             azure_policies=self.azure_policies,
             params_optional=params_optional,
             params_required=params_required,
             audit_only=False
         )
-        self.config = self.set_config(categorized_parameters=categorized_parameters)
+        self.parameters_config = self.set_parameter_config(categorized_parameters=categorized_parameters)
+
+    def json(self):
+        results = {}
+        for service_name, service_policies in self.parameters_config.items():
+            results[service_name] = {}
+            for policy_name, policy_parameters in service_policies.items():
+                results[service_name][policy_name] = []
+                for parameter_segment in policy_parameters:
+                    results[service_name][policy_name].append(parameter_segment.json())
+
+        return results
+
+    def __repr__(self):
+        return json.dumps(self.json())
 
-    def set_config(self, categorized_parameters: OverallCategorizedParameters) -> dict:
+    def set_parameter_config(self, categorized_parameters: CategorizedParameters) -> dict:
         results = {}
         for service_name, service_policies in categorized_parameters.service_categorized_parameters.items():
             results[service_name] = {}
@@ -55,8 +81,7 @@ def set_config(self, categorized_parameters: OverallCategorizedParameters) -> di
 
     def rendered(self) -> str:
         template_contents = dict(
-            # parameter_segments=parameter_segments,
-            categorized_parameters=self.config
+            categorized_parameters=self.parameters_config
         )
         template_path = os.path.join(os.path.dirname(__file__))
         env = Environment(loader=FileSystemLoader(template_path), lstrip_blocks=True)  # nosec

azure_guardrails/terraform/terraform_with_params.py

@@ -3,7 +3,7 @@
 import logging
 from jinja2 import Environment, FileSystemLoader
 from azure_guardrails.shared import utils
-from azure_guardrails.shared.parameters_categorized import OverallCategorizedParameters
+from azure_guardrails.shared.parameters_categorized import CategorizedParameters
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +14,7 @@ def __init__(
             self,
             policy_id_pairs: dict,
             parameter_requirement_str: str,
-            categorized_parameters: OverallCategorizedParameters,
+            categorized_parameters: CategorizedParameters,
             subscription_name: str = "",
             management_group: str = "",
             enforcement_mode: bool = False,

setup.cfg

@@ -1,10 +1,10 @@
 [nosetests]
 exe = True
-tests = test/, test/command/, test/guardrails, test/shared, test/terraform
+tests = test/, test/command/, test/iam_definition, test/shared, test/templates, test/terraform
 verbosity=2
 
 [tool:pytest]
-testpaths = test test/command test/guardrails test/shared test/terraform
+testpaths = test test/command test/iam_definition test/shared test/templates test/terraform
 python_files=test/*/test_*.py
 norecursedirs = .svn _build tmp* __pycache__