Merge pull request #180 from DontShaveTheYak/develop

Release v0.6.0

.github/workflows/release.yml

@@ -17,12 +17,12 @@ jobs:
     steps:
 
       - name: Checkout Code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.3
         with:
           fetch-depth: 0
 
       - name: Setup Python 3.9
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.1
         with:
           python-version: '3.9'
           architecture: x64
@@ -72,10 +72,10 @@ jobs:
     needs: tag
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.3
 
       - name: Setup Python 3.9
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.1
         with:
           python-version: '3.9'
           architecture: x64
@@ -104,7 +104,7 @@ jobs:
     needs: [tag, publish]
     steps:
       - name: Create Draft Release
-        uses: release-drafter/release-drafter@v5.23.0
+        uses: release-drafter/release-drafter@v5.24.0
         if: github.base_ref == 'develop'
         with:
           tag: ${{needs.tag.outputs.tag}}
@@ -114,7 +114,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create Release
-        uses: release-drafter/release-drafter@v5.23.0
+        uses: release-drafter/release-drafter@v5.24.0
         if: github.base_ref == 'master'
         with:
           tag: ${{needs.tag.outputs.tag}}

.github/workflows/test.yml

@@ -15,10 +15,10 @@ jobs:
     name: Python 3.9
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.3
 
       - name: Setup Latest Python
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.1
         with:
           python-version: 3.9
           architecture: x64
@@ -41,7 +41,7 @@ jobs:
         run: coverage xml --fail-under=0
 
       - name: Upload coverage to Codecov
-        uses: codecov/[email protected].1
+        uses: codecov/[email protected].4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: unit
@@ -52,14 +52,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.10"] # 3.9 is done in the job above
+        python-version: ["3.8", "3.10"] # 3.9 is done in the job above
     name: Python ${{ matrix.python-version }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.3
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.1
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64

.scripts/requirements.txt

@@ -1,2 +1,2 @@
-requests==2.28.2
-semver==2.13.0
+requests==2.31.0
+semver==3.0.1

pyproject.toml

@@ -12,7 +12,7 @@ license = "GPL-3.0-only"
 cf2tf = "cf2tf.app:cli"
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.8.1"
 cfn-flip = "^1.3.0"
 PyYAML = "^6.0"
 click = "^8.1.2"
@@ -24,12 +24,11 @@ pytest = "^7.1.2"
 
 [tool.poetry.dev-dependencies]
 black = "^23.0.0"
-flake8 = "^5.0.4"
+flake8 = "^6.0.0"
 flake8-black = "^0.3.3"
 coverage = {extras = ["toml"], version = "^7.0.0"}
 pytest-cov = "^4.0.0"
 mypy = "^1.1"
-typing-extensions = { version = "^4.3.0", python = "~3.7" }
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

src/cf2tf/app.py

@@ -14,7 +14,7 @@
 click_log.basic_config(log)
 
 
-@click.command()
+@click.command()  # type: ignore
 @click.version_option()
 @click.option("--output", "-o", type=click.Path(exists=False))
 @click_log.simple_verbosity_option(log)
@@ -48,4 +48,4 @@ def cli(output: Optional[str], template_path: str):
 
 
 if __name__ == "__main__":
-    cli()
+    cli()  # type: ignore

src/cf2tf/conversion/expressions.py

@@ -760,7 +760,7 @@ def sub_l(template: "TemplateConverter", values: List):
             "Fn::Sub - The first value must be a String and the second a Map."
         )
 
-    def replace_var(m):
+    def replace_var(m) -> str:
         var: str = m.group(1)
 
         if var in local_vars:
@@ -828,10 +828,11 @@ def ref(template: "TemplateConverter", var_name: str):
     if cf_resource:
         cf_resource_type = cf_resource.get("Type", "")
         docs_path = template.search_manager.find(cf_resource_type)
-        _, valid_attributes = doc_file.parse_attributes(docs_path)
+        valid_arguments, valid_attributes = doc_file.parse_attributes(docs_path)
         tf_name = cf2tf.convert.pascal_to_snake(var_name)
         tf_type = cf2tf.convert.create_resource_type(docs_path)
-        first_attr = next(iter(valid_attributes))
+
+        first_attr = valid_attributes[0] if valid_attributes else valid_arguments[0]
         conditional = cf_resource.get("Condition")
 
         if conditional is not None:
@@ -976,7 +977,11 @@ def wrap_in_curlys(input: str):
 # functions that are allowed to be nested inside it.
 ALLOWED_FUNCTIONS: Dict[str, Dispatch] = {
     "Fn::And": ALLOWED_NESTED_CONDITIONS,
-    "Fn::Equals": {**ALLOWED_NESTED_CONDITIONS, "Fn::Join": join},
+    "Fn::Equals": {
+        **ALLOWED_NESTED_CONDITIONS,
+        "Fn::Join": join,
+        "Fn::Select": select,
+    },
     "Fn::If": {
         "Fn::Base64": base64,
         "Fn::FindInMap": find_in_map,

src/cf2tf/conversion/overrides.py

@@ -2,6 +2,7 @@
 
 from cf2tf.terraform.hcl2.custom import LiteralType
 from cf2tf.terraform.hcl2.primitive import NullType, StringType, TerraformType
+from cf2tf.terraform.hcl2.complex import ListType, MapType
 
 if TYPE_CHECKING:
     from cf2tf.convert import TemplateConverter
@@ -45,7 +46,23 @@ def s3_bucket_policy(_tc: "TemplateConverter", params: CFParams) -> CFParams:
     return params
 
 
+def tag_conversion(_tc: "TemplateConverter", params: CFParams) -> CFParams:
+    if isinstance(params["Tags"], dict):
+        return params
+
+    orginal_tags: ListType = params["Tags"]  # type: ignore
+
+    new_tags = {LiteralType(tag["Key"]): tag["Value"] for tag in orginal_tags}
+
+    del params["Tags"]
+    params["tags"] = MapType(new_tags)
+
+    return params
+
+
 OVERRIDE_DISPATCH: ResourceOverride = {
     "aws_s3_bucket": {"AccessControl": s3_bucket_acl},
     "aws_s3_bucket_policy": {"PolicyDocument": s3_bucket_policy},
 }
+
+GLOBAL_OVERRIDES: ParamOverride = {"Tags": tag_conversion}

src/cf2tf/convert.py

@@ -10,7 +10,7 @@
 import cf2tf.conversion.expressions as functions
 import cf2tf.terraform._configuration as config
 import cf2tf.terraform.doc_file as doc_file
-from cf2tf.conversion.overrides import OVERRIDE_DISPATCH
+from cf2tf.conversion.overrides import OVERRIDE_DISPATCH, GLOBAL_OVERRIDES
 from cf2tf.terraform.blocks import Block, Locals, Output, Resource, Variable
 from cf2tf.terraform.hcl2 import AllTypes
 from cf2tf.terraform.hcl2.complex import ListType, MapType
@@ -140,7 +140,7 @@ def convert_to_tf(self, manifest: Manifest):
 
         return tf_resources
 
-    def resolve_values(
+    def resolve_values(  # noqa: max-complexity=13
         self,
         data: Any,
         allowed_func: functions.Dispatch,
@@ -181,7 +181,10 @@ def resolve_values(
                     value, functions.ALLOWED_FUNCTIONS[key], key, inside_function=True
                 )
 
-                return allowed_func[key](self, value)
+                try:
+                    return allowed_func[key](self, value)
+                except Exception:
+                    return CommentType(f"Unable to resolve {key} with value: {value}")
 
             return MapType(data)
         elif isinstance(data, list):
@@ -323,6 +326,10 @@ def convert_resources(self, resources: CFResources):
                     tf_type, resolved_values, self
                 )
 
+                overrided_values = perform_global_overrides(
+                    tf_type, overrided_values, self
+                )
+
                 log.debug("Converting property names to argument names...")
 
                 arguments = props_to_args(overrided_values, valid_arguments, docs_path)
@@ -603,3 +610,15 @@ def perform_resource_overrides(
             params = override(tc, params)
 
     return params
+
+
+def perform_global_overrides(
+    tf_type: str, params: Dict[str, TerraformType], tc: TemplateConverter
+):
+    log.debug("Performing global overrides for {tf_type}")
+
+    for param, override in GLOBAL_OVERRIDES.items():
+        if param in params:
+            params = override(tc, params)
+
+    return params

src/cf2tf/terraform/code.py

@@ -3,11 +3,12 @@
 from pathlib import Path
 from tempfile import gettempdir
 from typing import Optional
+from shutil import rmtree
 
 import click
 from click._termui_impl import ProgressBar
 from git import RemoteProgress
-from git.repo.base import Repo
+from git.repo.base import Repo, InvalidGitRepositoryError
 from thefuzz import fuzz, process  # type: ignore
 
 import cf2tf.convert
@@ -34,7 +35,7 @@ def find(self, resource_type: str) -> Path:
         ranking: int
         doc_path: Path
         resource_name, ranking, doc_path = process.extractOne(
-            name.lower(), files, scorer=fuzz.UWRatio
+            name.lower(), files, scorer=fuzz.token_sort_ratio
         )
 
         log.debug(
@@ -57,16 +58,19 @@ def search_manager():
     return SearchManager(docs_path)
 
 
-def get_code():
+def get_code() -> Repo:
     temp_dir = Path(gettempdir())
     repo_path = temp_dir.joinpath("terraform_src")
 
-    if repo_path.exists():
-        if repo_path.joinpath(".git").exists():
-            # todo Need to check to make sure the remote is correct
-            click.echo(" existing repo found.")
-            repo = Repo(repo_path)
-            return repo
+    try:
+        existing_repo = repo_from_existing(repo_path)
+
+        if existing_repo:
+            return existing_repo
+
+    except InvalidGitRepositoryError:
+        rmtree(repo_path)
+        repo_path.rmdir()
 
     print(f"// Cloning Terraform src code to {repo_path}...", end="")
 
@@ -76,13 +80,23 @@ def get_code():
         "https://github.com/hashicorp/terraform-provider-aws.git",
         repo_path,
         depth=1,
-        progress=CloneProgress(),
+        progress=CloneProgress(),  # type: ignore
     )
     click.echo(" code has been checked out.")
 
     return repo
 
 
+def repo_from_existing(repo_path: Path) -> Optional[Repo]:
+    if repo_path.exists():
+        if repo_path.joinpath(".git").exists():
+            repo = Repo(repo_path)
+            click.echo(f"// Existing Terraform src code found at {repo_path}.")
+            return repo
+
+    return None
+
+
 def resource_type_to_name(resource_type: str) -> str:
     """Converts a Cloudformation Resource Type into something more search friendly.
 

src/cf2tf/terraform/doc_file.py

@@ -10,9 +10,15 @@
 
 def parse_attributes(docs_path: Path):
     with open(docs_path) as file:
-        arguments = parse_section("Argument Reference", file)
+        try:
+            arguments = parse_section("Argument Reference", file)
+        except Exception as e:
+            raise Exception(f"Unable to find arguments in {file.name}") from e
 
-        attributes = parse_section("Attributes Reference", file)
+        try:
+            attributes = parse_section("Attributes Reference", file)
+        except Exception as e:
+            raise Exception(f"Unable to find attributes in {file.name}") from e
 
     return (arguments, attributes)
 
@@ -61,7 +67,7 @@ def parse_items(file: TextIOWrapper):
 
         # These should be the attributes we are after
         if line[0] == "*":
-            regex = r"`([\w\.]+)`"
+            regex = r"`([\w.*]+)`"
 
             match = re.search(regex, line)
 

src/cf2tf/terraform/hcl2/primitive.py

@@ -18,7 +18,7 @@ class TerraformType(Protocol):
     value: Any
 
     def __str__(self) -> str:
-        return self.render()
+        return self.render(0)
 
     @abstractclassmethod
     def render(self, indent: int) -> str:

tests/test_convert.py

@@ -153,6 +153,22 @@ def test_perform_resource_overrides():
     assert "private" == result["acl"]
 
 
+def test_perform_global_overrides():
+    template = tc()
+
+    params = {"Tags": [{"Key": "foo", "Value": "bar"}]}
+
+    result = convert.perform_global_overrides("aws_s3_bucket", params, template)
+
+    assert result is params
+
+    assert "Tags" not in result
+    assert "tags" in result
+    assert "foo" in result["tags"]
+    assert "bar" == result["tags"]["foo"]
+    assert isinstance(result["tags"], dict)
+
+
 parse_subsection_tests = [
     # (tf_arg_name, cf_props, docs_path, expected_type, expectation)
     (