From 887aaa73a94ebcdd437aa4c70416337a10cb0e64 Mon Sep 17 00:00:00 2001 From: osherElm <52105667+osherElm@users.noreply.github.com> Date: Sun, 5 Feb 2023 10:53:06 +0200 Subject: [PATCH] feat/remote repository support (#34) - Add the ability to support Remote repositories - Added support for multiple git resources with private repository support ( Will change token on each clone request to the token for the specific git provider) - Github - Gitlab - Added tests for each of the new providers added - Added tests for Template parser - Added tests for git provider - Added Readme to update the new Remote feature --- .gitignore | 3 + Makefile | 1 - README.md | 63 ++++++++ examples/templates/main.tf.tmpl | 28 ++++ go.mod | 19 +++ go.sum | 22 +++ internal/cmd/app/generate_all.go | 28 +++- internal/cmd/app/generate_main.go | 31 ++++ internal/cmd/app/initializer.go | 8 +- internal/cmd/types/terraform_variable.go | 1 + .../drivers/{ => parser}/terraform_parser.go | 5 +- .../{ => parser}/terraform_parser_test.go | 16 +- .../template_reader/template_remote_module.go | 140 ++++++++++++++++++ .../template_remote_module_test.go | 49 ++++++ .../drivers/template_reader/test.tmpl | 19 +++ .../services/drivers/version_control/git.go | 135 +++++++++++++++++ .../drivers/version_control/git_test.go | 122 +++++++++++++++ internal/services/template_api.go | 10 ++ .../templates/variables_modules_tf.go | 2 +- internal/services/terraform.go | 25 ++-- 20 files changed, 698 insertions(+), 29 deletions(-) rename internal/services/drivers/{ => parser}/terraform_parser.go (97%) rename internal/services/drivers/{ => parser}/terraform_parser_test.go (79%) create mode 100644 internal/services/drivers/template_reader/template_remote_module.go create mode 100644 internal/services/drivers/template_reader/template_remote_module_test.go create mode 100644 internal/services/drivers/template_reader/test.tmpl create mode 100644 internal/services/drivers/version_control/git.go create mode 100644 internal/services/drivers/version_control/git_test.go diff --git a/.gitignore b/.gitignore index 9fbf3b8..3252f45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ main main.tmpl config bin/* +.idea +*.DS_Store +.DS_Store \ No newline at end of file diff --git a/Makefile b/Makefile index 27ab455..11f25b5 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,6 @@ get-linter: ## Get golangci-lint lint: get-linter ## Run linter $(info) $(info ========[ $@ ]========) - rm lint.xml golangci-lint run test: ## Run go tests diff --git a/README.md b/README.md index daad154..cbd055c 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,69 @@ terra-crust terraform-all --destination-path="." --source-path=".../modules" * same as Main has additional flag: ``main-template-path`` to support external main templates like in examples/templates/main.tf.tmpl * Upon failing on create one of the files , It wont fail the entire flow(Will keep on to the next files). +## Remote TerraCrust: + +### Description +From version 2.0.0 TerraCrust will support fetching remote modules +and composite them into the root module that is being created. +### How to use +in order to activate this feature all you have to do is to set `fetch-remote` to true like so: +```terra-crust terraform-all --main-template-path=./terraform/main.tmpl --destination-path="." --source-path=".../modules" fetch-remote=true``` +When activating `FetchRemote` you must use Custom main template, terracrust will look for all sources +that are from git it will look for this pattern: +1. `"git::REPOSITORY.git/PATH/MODULE"` +2. `"git::REPOSITORY"` +it will start with git:: then the repository has to end with .git if it has +more than 1 module inside it, if the repo is just the root module then no need to add .git to the end. +it will also support versioning/ branch reference so feel free to add them , +more examples could be found under `examples/templates`. + +Output Example for this module: +```"git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account"``` +``` +module "iam-account" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" + + # account_alias = module. TODO: Add Required Field + + + hard_expiry = local.iam-account.hard_expiry + require_numbers = local.iam-account.require_numbers + require_symbols = local.iam-account.require_symbols + create_account_password_policy = local.iam-account.create_account_password_policy + allow_users_to_change_password = local.iam-account.allow_users_to_change_password + minimum_password_length = local.iam-account.minimum_password_length + password_reuse_prevention = local.iam-account.password_reuse_prevention + require_lowercase_characters = local.iam-account.require_lowercase_characters + require_uppercase_characters = local.iam-account.require_uppercase_characters + get_caller_identity = local.iam-account.get_caller_identity + max_password_age = local.iam-account.max_password_age + +} +``` + +this will propagate all the default and required variables using the Templates Api + +## Template API: +The main.tmpl exposing Template API that includes for now 2 functions: +1. GetRequired - Will expose the require variables with option for you to fill +2. GetDefaults - Will expose the optional variables - without needing to fill them up + +For example: +``` +module "iam-account" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" + + {{(GetRequired "iam-account" .)}} + + {{(GetDefaults "iam-account" .)}} +} +``` + +Results can be seen at the section above, basically after you fill the required variables, +you want to get rid of `{{(GetRequired "iam-account" .)}}` because it will keep overwriting your changes +so once you filled the required variables you can drop it instead and put the required variables you filled in the template. + ## Example Usage Further guidelines as well as examples, can be shown [here](./examples/example.md). diff --git a/examples/templates/main.tf.tmpl b/examples/templates/main.tf.tmpl index b724468..12f414a 100644 --- a/examples/templates/main.tf.tmpl +++ b/examples/templates/main.tf.tmpl @@ -18,4 +18,32 @@ module "zookeeper" { # Configurable Variables - Optinal Fields {{(GetDefaults "consul_sync" .)}} +} + +module "zones" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-route53.git/modules/zones" + + # Dependency Injection - Required Fields + {{(GetRequired "zones" .)}} + + # Configurable Variables - Optinal Fields + {{(GetDefaults "zones" .)}} +} + +module "terraform-aws-resource-naming" { + source = "git::https://github.com/traveloka/terraform-aws-resource-naming?ref=v0.23.1" + + # Dependency Injection - Required Fields + {{(GetRequired "terraform-aws-resource-naming" .)}} + + # Configurable Variables - Optinal Fields + {{(GetDefaults "terraform-aws-resource-naming" .)}} +} + +module "iam-account" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" + + {{(GetRequired "iam-account" .)}} + + {{(GetDefaults "iam-account" .)}} } \ No newline at end of file diff --git a/go.mod b/go.mod index 1794121..d80330e 100644 --- a/go.mod +++ b/go.mod @@ -4,23 +4,42 @@ go 1.19 require ( github.com/AppsFlyer/go-logger v1.1.1 + github.com/go-git/go-git/v5 v5.4.2 + github.com/go-test/deep v1.0.3 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl/v2 v2.15.0 github.com/hashicorp/terraform-exec v0.17.3 + github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.6.1 github.com/zclconf/go-cty v1.12.1 ) require ( + github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/emirpasic/gods v1.12.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/terraform-json v0.14.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect + github.com/otiai10/copy v1.9.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xanzy/ssh-agent v0.3.0 // indirect golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/text v0.3.7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 0456c01..aea3dde 100644 --- a/go.sum +++ b/go.sum @@ -9,10 +9,12 @@ github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -24,12 +26,14 @@ github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= @@ -37,6 +41,7 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -86,6 +91,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -99,8 +105,16 @@ github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= +github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg= @@ -108,6 +122,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -120,6 +135,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= @@ -147,6 +163,7 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -162,6 +179,9 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -177,6 +197,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -190,4 +211,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/app/generate_all.go b/internal/cmd/app/generate_all.go index 0048c8a..d3145fc 100644 --- a/internal/cmd/app/generate_all.go +++ b/internal/cmd/app/generate_all.go @@ -15,8 +15,11 @@ package app import ( - "github.com/AppsFlyer/terra-crust/internal/cmd/types" "github.com/spf13/cobra" + + "github.com/AppsFlyer/terra-crust/internal/cmd/types" + template_reader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" + version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" ) func generateAllFiles(root *RootCommand) *cobra.Command { @@ -27,6 +30,28 @@ func generateAllFiles(root *RootCommand) *cobra.Command { Example: "", RunE: func(cmd *cobra.Command, args []string) error { log := root.log + templateReader := template_reader.InitTemplateRemoteModule(log) + gitDriver := version_control.InitGitProvider(log) + + if flags.FetchRemote && flags.MainTemplateFilePath != "" { + remoteModulesMap, err := templateReader.GetRemoteModulesFromTemplate(flags.MainTemplateFilePath) + if err != nil { + log.Error("Failed parsing remote modules from custom template", err.Error()) + + return err + } + + if err = gitDriver.CloneModules(remoteModulesMap, flags.SourcePath); err != nil { + log.Error("Failed cloning remote modules ", err.Error()) + + return err + } + defer func() { + if err = gitDriver.CleanModulesFolders(remoteModulesMap, flags.SourcePath); err != nil { + log.Errorf("Failed to clean up some of the remote resources please make sure to clean it manually and check the error , %s", err.Error()) + } + }() + } terraformSvc := InitTerraformGeneratorService(log) @@ -55,6 +80,7 @@ func generateAllFiles(root *RootCommand) *cobra.Command { cmd.Flags().StringVar(&flags.SourcePath, "source-path", "", "Required: General module folder path that contains all the sub modules flattened") cmd.Flags().StringVar(&flags.DestinationPath, "destination-path", "", "Required: Destination path to write the new terraform file") cmd.Flags().StringVar(&flags.MainTemplateFilePath, "main-template-path", "", "Optional: Custom main template path for generated module, will take default if not provided") + cmd.Flags().BoolVar(&flags.FetchRemote, "fetch-remote", false, "Optional: Enable fetching of remote modules and exporting their variables in root module") if err := cmd.MarkFlagRequired("source-path"); err != nil { root.log.Error("failed to set required flag on source-path", err.Error()) } diff --git a/internal/cmd/app/generate_main.go b/internal/cmd/app/generate_main.go index fafb703..9ad3859 100644 --- a/internal/cmd/app/generate_main.go +++ b/internal/cmd/app/generate_main.go @@ -16,6 +16,8 @@ package app import ( "github.com/AppsFlyer/terra-crust/internal/cmd/types" + template_reader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" + version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" "github.com/spf13/cobra" ) @@ -29,6 +31,34 @@ func generateMain(root *RootCommand) *cobra.Command { log := root.log terraformSvc := InitTerraformGeneratorService(log) + templateReader := template_reader.InitTemplateRemoteModule(log) + gitDriver := version_control.InitGitProvider(log) + + if flags.FetchRemote && flags.MainTemplateFilePath != "" { + log.Infof("Searching for remote modules") + remoteModulesMap, err := templateReader.GetRemoteModulesFromTemplate(flags.MainTemplateFilePath) + if err != nil { + log.Error("Failed parsing remote modules from custom template", err.Error()) + + return err + } + + log.Infof("found remote modules: ", remoteModulesMap) + + if err = gitDriver.CloneModules(remoteModulesMap, flags.SourcePath); err != nil { + log.Error("Failed cloning remote modules ", err.Error()) + + return err + } + + defer func() { + if err = gitDriver.CleanModulesFolders(remoteModulesMap, flags.SourcePath); err != nil { + log.Errorf("Failed to clean up some of the remote resources please make sure to clean it manually and check the error , %s", err.Error()) + } + }() + } + + log.Infof("remote not found ") if err := terraformSvc.GenerateMain(flags.SourcePath, flags.DestinationPath, flags.MainTemplateFilePath); err != nil { log.Error("Failed generating the terraform main file", err.Error()) @@ -43,6 +73,7 @@ func generateMain(root *RootCommand) *cobra.Command { cmd.Flags().StringVar(&flags.SourcePath, "source-path", "", "Required: General module folder path that contains all the sub modules flattened") cmd.Flags().StringVar(&flags.DestinationPath, "destination-path", "", "Required: Destination path to write the new terraform file") cmd.Flags().StringVar(&flags.MainTemplateFilePath, "main-template-path", "", "Optional: Custom main template path for generated module, will take default if not provided") + cmd.Flags().BoolVar(&flags.FetchRemote, "fetch-remote", false, "Optional: Enable fetching of remote modules and exporting their variables in root module") if err := cmd.MarkFlagRequired("source-path"); err != nil { root.log.Error("failed to set required flag on source-path", err.Error()) } diff --git a/internal/cmd/app/initializer.go b/internal/cmd/app/initializer.go index b0e88c9..d24d850 100644 --- a/internal/cmd/app/initializer.go +++ b/internal/cmd/app/initializer.go @@ -17,7 +17,7 @@ package app import ( logger "github.com/AppsFlyer/go-logger" "github.com/AppsFlyer/terra-crust/internal/services" - "github.com/AppsFlyer/terra-crust/internal/services/drivers" + "github.com/AppsFlyer/terra-crust/internal/services/drivers/parser" ) const ( @@ -27,11 +27,11 @@ const ( ) func InitTerraformGeneratorService(log logger.Logger) *services.Terraform { - parserDriver := drivers.NewTerraformParser(log) - parser := services.NewParser(log, parserDriver) + parserDriver := parser.NewTerraformParser(log) + parserSvc := services.NewParser(log, parserDriver) templateHandler := services.NewTemplateHandler(log) - tfSvc := services.NewTerraform(log, parser, templateHandler, LocalsTemplatePath, VariableTemplatePath, MainTemplatePath) + tfSvc := services.NewTerraform(log, parserSvc, templateHandler, LocalsTemplatePath, VariableTemplatePath, MainTemplatePath) return tfSvc } diff --git a/internal/cmd/types/terraform_variable.go b/internal/cmd/types/terraform_variable.go index 27abe39..6af8a37 100644 --- a/internal/cmd/types/terraform_variable.go +++ b/internal/cmd/types/terraform_variable.go @@ -18,4 +18,5 @@ type TFGenerateFlags struct { SourcePath string DestinationPath string MainTemplateFilePath string + FetchRemote bool } diff --git a/internal/services/drivers/terraform_parser.go b/internal/services/drivers/parser/terraform_parser.go similarity index 97% rename from internal/services/drivers/terraform_parser.go rename to internal/services/drivers/parser/terraform_parser.go index d69cc81..6f3b024 100644 --- a/internal/services/drivers/terraform_parser.go +++ b/internal/services/drivers/parser/terraform_parser.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package drivers +package parser import ( "errors" + "github.com/AppsFlyer/terra-crust/internal/services/drivers" "os" "path/filepath" "strings" @@ -38,7 +39,7 @@ type TerraformParser struct { logger logger.Logger } -func NewTerraformParser(log logger.Logger) Parser { +func NewTerraformParser(log logger.Logger) drivers.Parser { return &TerraformParser{ logger: log, } diff --git a/internal/services/drivers/terraform_parser_test.go b/internal/services/drivers/parser/terraform_parser_test.go similarity index 79% rename from internal/services/drivers/terraform_parser_test.go rename to internal/services/drivers/parser/terraform_parser_test.go index ffa836d..3d06c42 100644 --- a/internal/services/drivers/terraform_parser_test.go +++ b/internal/services/drivers/parser/terraform_parser_test.go @@ -12,21 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package drivers_test +package parser_test import ( + "github.com/AppsFlyer/terra-crust/internal/services/drivers/parser" "testing" logger "github.com/AppsFlyer/go-logger" - "github.com/AppsFlyer/terra-crust/internal/services/drivers" ) func TestParse(t *testing.T) { t.Parallel() log := logger.NewSimple() - parser := drivers.NewTerraformParser(log) + parserDriver := parser.NewTerraformParser(log) - m, err := parser.Parse("../../../mock/modules") + m, err := parserDriver.Parse("../../../../mock/modules") if err != nil { t.Errorf("failed to parse , reason: %s", err.Error()) } @@ -47,9 +47,9 @@ func TestParse(t *testing.T) { func TestParseBadPath(t *testing.T) { t.Parallel() log := logger.NewSimple() - parser := drivers.NewTerraformParser(log) + parserDriver := parser.NewTerraformParser(log) - m, err := parser.Parse("../../../internal") + m, err := parserDriver.Parse("../../../../internal") if err != nil { t.Errorf("failed to parse , reason: %s", err.Error()) } @@ -62,9 +62,9 @@ func TestParseBadPath(t *testing.T) { func TestParseNotExistingPath(t *testing.T) { t.Parallel() log := logger.NewSimple() - parser := drivers.NewTerraformParser(log) + parserDriver := parser.NewTerraformParser(log) - _, err := parser.Parse("../../internal") + _, err := parserDriver.Parse("../../../internal") if err == nil { t.Errorf("Expected error for bad route") } diff --git a/internal/services/drivers/template_reader/template_remote_module.go b/internal/services/drivers/template_reader/template_remote_module.go new file mode 100644 index 0000000..d533430 --- /dev/null +++ b/internal/services/drivers/template_reader/template_remote_module.go @@ -0,0 +1,140 @@ +package template_reader + +import ( + "errors" + version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" + "io" + "os" + "regexp" + "strings" + + log "github.com/AppsFlyer/go-logger" +) + +const ( + gitSourceLineRegex = "source\\s*=\\s*\"git::" + moduleNameRegex = `.*\/([^?]+)` + gitUrlRegex = `(https?:\/\/.+\/.+\.git)` + gitUrlRegexVersion = `(https?:\/\/[^/]+\/[^?]+)` + versionRegex = `/?ref=(.*)"` + modulePathRegex = `.git[\/]+([^\/][^\?]+)[\?]{0,1}.*["$]` +) + +type TemplateRemoteModule struct { + log log.Logger + modules map[string]*version_control.RemoteModule +} + +func InitTemplateRemoteModule(log log.Logger) *TemplateRemoteModule { + return &TemplateRemoteModule{ + log: log, + modules: make(map[string]*version_control.RemoteModule), + } +} + +func (trm *TemplateRemoteModule) GetRemoteModulesFromTemplate(templatePath string) (map[string]*version_control.RemoteModule, error) { + file, err := os.Open(templatePath) + if err != nil { + trm.log.Error("Failed to open template to fetch remote modules,make sure you have a custom template when using this feature") + + return nil, err + } + defer func() { + err := file.Close() + if err != nil { + trm.log.Errorf("Failed to close properly the main tmpl , error: %s", err.Error()) + } + }() + + b, err := io.ReadAll(file) + if err != nil { + trm.log.Error("Failed to read file") + + return nil, err + } + + r, err := regexp.Compile(gitSourceLineRegex) + if err != nil { + trm.log.Error("Failed to compile regex") + + return nil, err + } + + lines := strings.Split(string(b), "\n") + for _, line := range lines { + if !r.MatchString(line) { + continue + } + + remoteModule, err := trm.parseLineIntoRemoteModule(line) + if err != nil { + trm.log.Error("Failed to parse remote Source value from template") + + return nil, err + } + + trm.modules[remoteModule.Name] = remoteModule + } + + return trm.modules, nil +} + +func (trm *TemplateRemoteModule) parseLineIntoRemoteModule(line string) (*version_control.RemoteModule, error) { + re, err := regexp.Compile(moduleNameRegex) + if err != nil { + trm.log.Errorf("failed to compile regex: %s ", moduleNameRegex) + + return nil, err + } + + match := re.FindStringSubmatch(line) + if len(match) <= 1 { + return nil, errors.New("failed to fetch module Name from source") + } + moduleName := strings.TrimSuffix(match[1], "\"") + + urlReg, err := regexp.Compile(gitUrlRegex) + if err != nil { + trm.log.Errorf("failed to compile regex: %s ", gitUrlRegex) + + return nil, err + } + + urlVersionReg, err := regexp.Compile(gitUrlRegexVersion) + if err != nil { + trm.log.Errorf("failed to compile regex: %s ", gitUrlRegexVersion) + + return nil, err + } + + urlMatch := urlReg.FindStringSubmatch(line) + urlVersionMatch := urlVersionReg.FindStringSubmatch(line) + + if len(urlMatch) < 1 && len(urlVersionMatch) < 1 { + return nil, errors.New("failed to fetch module URL from source") + } + + versReg := regexp.MustCompile(versionRegex) + verMatch := versReg.FindStringSubmatch(line) + version := "" + if len(verMatch) >= 1 { + version = verMatch[1] + } + + modulePathReg := regexp.MustCompile(modulePathRegex) + modulePathMatch := modulePathReg.FindStringSubmatch(line) + modulePath := "" + if len(modulePathMatch) >= 1 { + modulePath = modulePathMatch[1] + } + + if len(urlMatch) >= 1 { + gitUrl := strings.TrimSuffix(urlMatch[1], `"`) + + return &version_control.RemoteModule{Name: moduleName, Url: gitUrl, Version: version, Path: modulePath}, nil + } + + gitUrl := strings.TrimSuffix(urlVersionMatch[1], `"`) + + return &version_control.RemoteModule{Name: moduleName, Url: gitUrl, Version: version, Path: modulePath}, nil +} diff --git a/internal/services/drivers/template_reader/template_remote_module_test.go b/internal/services/drivers/template_reader/template_remote_module_test.go new file mode 100644 index 0000000..c6dede1 --- /dev/null +++ b/internal/services/drivers/template_reader/template_remote_module_test.go @@ -0,0 +1,49 @@ +package template_reader_test + +import ( + version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" + "github.com/go-test/deep" + "testing" + + logger "github.com/AppsFlyer/go-logger" + tmplReader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" +) + +var result = map[string]*version_control.RemoteModule{ + "terra-crust": {Name: "terra-crust", Url: "https://github.com/AppsFlyer/terra-crust", Version: "", Path: ""}, + "naming": {Name: "naming", Url: "https://github.domain.com/test/terraform/modules/naming.git", Version: "0.2.1", Path: "modules/naming"}, + "otel-collector": {Name: "otel-collector", Url: "https://github.com/streamnative/terraform-helm-charts.git", Version: "v0.2.1", Path: "modules/otel-collector"}, + "iam-account": {Name: "iam-account", Path: "modules/iam-account", Url: "https://github.com/terraform-aws-modules/terraform-aws-iam.git", Version: ""}, + "zones": {Name: "zones", Path: "modules/zones", Url: "https://github.com/terraform-aws-modules/terraform-aws-route53.git", Version: ""}, +} + +func TestGetRemoteModulesFromTemplate(t *testing.T) { + t.Parallel() + log := logger.NewSimple() + templateReader := tmplReader.InitTemplateRemoteModule(log) + + modules, err := templateReader.GetRemoteModulesFromTemplate("./test.tmpl") + if err != nil { + t.Errorf("failed to extract sources from template %s", err.Error()) + } + + if len(modules) != 5 { + t.Errorf("Expected 5 modules found : %d", len(modules)) + } + + if diff := deep.Equal(modules, result); diff != nil { + t.Errorf("expected result to be equal, result is different : %s", diff) + } +} + +func TestBadPath(t *testing.T) { + t.Parallel() + log := logger.NewSimple() + templateReader := tmplReader.InitTemplateRemoteModule(log) + + _, err := templateReader.GetRemoteModulesFromTemplate("./main.tmpl") + if err == nil { + t.Errorf("expected to have error for no file found") + } + +} diff --git a/internal/services/drivers/template_reader/test.tmpl b/internal/services/drivers/template_reader/test.tmpl new file mode 100644 index 0000000..5b9d403 --- /dev/null +++ b/internal/services/drivers/template_reader/test.tmpl @@ -0,0 +1,19 @@ +module "terra-crust" { + source = "git::https://github.com/AppsFlyer/terra-crust" +} + +module "naming" { + source = "git::https://github.domain.com/test/terraform/modules/naming.git//modules/naming?ref=0.2.1" +} + +module "otel_collector" { + source = "git::https://github.com/streamnative/terraform-helm-charts.git//modules/otel-collector?ref=v0.2.1" +} + +module "iam-account" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git//modules/iam-account" +} + +module "zones" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-route53.git/modules/zones" +} \ No newline at end of file diff --git a/internal/services/drivers/version_control/git.go b/internal/services/drivers/version_control/git.go new file mode 100644 index 0000000..ce81000 --- /dev/null +++ b/internal/services/drivers/version_control/git.go @@ -0,0 +1,135 @@ +package version_control + +import ( + "fmt" + log "github.com/AppsFlyer/go-logger" + "github.com/go-git/go-git/v5" /// with go modules disabled + "github.com/go-git/go-git/v5/plumbing/transport/http" + cp "github.com/otiai10/copy" + "github.com/pkg/errors" + "os" + "strings" +) + +const ( + FolderPathFormat = "%s/%s" + TempFolderPath = "%s/temp_clone_path/%s" + GitlabTokenENV = "" + GithubTokenENV = "" + GitlabUserENV = "" + GithubUserENV = "" +) + +type RemoteModule struct { + Name string + Url string + Version string + Path string +} + +type Git struct { + log log.Logger +} + +func InitGitProvider(log log.Logger) *Git { + return &Git{ + log: log, + } +} + +func (g *Git) CloneModules(modules map[string]*RemoteModule, modulesSource string) error { + for moduleName, moduleData := range modules { + clonePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) + if moduleData.Path != "" { + clonePath = fmt.Sprintf(TempFolderPath, modulesSource, moduleName) + } + + if err := g.clone(moduleData, clonePath); err != nil { + return errors.Wrapf(err, "failed to fetch module , url: %s", moduleData.Url) + } + + g.log.Infof("Copy folder from : %s/%s to : %s", clonePath, moduleData.Path, modulesSource) + + if moduleData.Path != "" { + modulePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) + + err := cp.Copy(fmt.Sprintf(FolderPathFormat, clonePath, moduleData.Path), modulePath) + if err != nil { + g.log.Errorf("failed to copy desired terraform module module path :%s, module name: %s", clonePath, moduleData.Name) + } + + g.log.Infof("Changing module path from : %s to : %s", modules[moduleName].Path, modulePath) + modules[moduleName].Path = moduleName + } + } + + if err := g.cleanTemp(modulesSource); err != nil { + return errors.Wrap(err, "Failed to delete temp folder") + } + + return nil +} + +func (g *Git) clone(moduleData *RemoteModule, directoryPath string) error { + userName, token := g.getGitUserNameAndToken(moduleData.Url) + + remoteName := "origin" + if moduleData.Version != "" { + remoteName = moduleData.Version + } + + _, err := git.PlainClone(directoryPath, false, &git.CloneOptions{ + URL: moduleData.Url, + Auth: &http.BasicAuth{Password: token, Username: userName}, + RemoteName: remoteName, + Depth: 1, + }) + + return err +} + +func (g *Git) CleanModulesFolders(modules map[string]*RemoteModule, modulesSource string) error { + var returnedErr error = nil + for moduleName := range modules { + modulePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) + + err := os.RemoveAll(modulePath) + if err != nil { + g.log.Errorf("Failed to clear up a module %s at path: %s , error:%s", moduleName, modulePath, err.Error()) + if returnedErr == nil { + returnedErr = err + } + + returnedErr = errors.Wrap(returnedErr, err.Error()) + } + } + + return returnedErr +} + +func (g *Git) cleanTemp(modulesSourcePath string) error { + tempPath := fmt.Sprintf("%s/temp_clone_path", modulesSourcePath) + + g.log.Infof("Deleting temp folder containing all clones") + + err := os.RemoveAll(tempPath) + if err != nil { + g.log.Errorf("Failed to clear up temp folder that been used for clone, please clean it manually , failed on temp folder, path : %s , err: %s", tempPath, err.Error()) + + return err + } + + return nil +} + +func (g *Git) getGitUserNameAndToken(url string) (string, string) { + if strings.Contains(url, "gitlab") { + return os.Getenv(GitlabUserENV), os.Getenv(GitlabTokenENV) + } + + if strings.Contains(url, "github") { + return os.Getenv(GithubUserENV), os.Getenv(GithubTokenENV) + } + + return "", "" +} diff --git a/internal/services/drivers/version_control/git_test.go b/internal/services/drivers/version_control/git_test.go new file mode 100644 index 0000000..30b0bf7 --- /dev/null +++ b/internal/services/drivers/version_control/git_test.go @@ -0,0 +1,122 @@ +package version_control_test + +import ( + logger "github.com/AppsFlyer/go-logger" + version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" + "os" + "testing" +) + +const ( + TerraCrustModuleName = "terracrust" + NamingModuleName = "terraform-aws-resource-naming" + ZonesModuleName = "zones" + TerraCrustURL = "https://github.com/AppsFlyer/terra-crust" + + ModulesTestPath = "./temp-git-test" +) + +var mockModules = map[string]*version_control.RemoteModule{ + TerraCrustModuleName: { + Url: TerraCrustURL, + }, + NamingModuleName: { + Url: "https://github.com/traveloka/terraform-aws-resource-naming", + Version: "v0.23.1", + }, + ZonesModuleName: { + Url: "https://github.com/terraform-aws-modules/terraform-aws-route53.git", + Path: "modules/zones", + }, +} +var mockBadModules = map[string]*version_control.RemoteModule{ + TerraCrustModuleName: { + Url: "https://github.com/appsflyer/terra-crust/test/bad", + }, +} + +func TestCloneAndCleanup(t *testing.T) { + t.Parallel() + log := logger.NewSimple() + + gitDriver := version_control.InitGitProvider(log) + + err := gitDriver.CloneModules(mockModules, ModulesTestPath) + if err != nil { + t.Errorf(err.Error()) + } + + folders, err := getFolderAsMap(ModulesTestPath) + if err != nil { + t.Errorf(err.Error()) + } + + if _, exist := folders[TerraCrustModuleName]; !exist { + t.Errorf("Clone failed , did not find terra-crust") + } + + if _, exist := folders[ZonesModuleName]; !exist { + t.Errorf("Clone failed , did not find zones") + } + + if _, exist := folders[NamingModuleName]; !exist { + t.Errorf("Clone failed , did not find naming") + } + + if len(folders) != 3 { + t.Errorf("Expected 3 folder count received %d", len(folders)) + } + + if err = gitDriver.CleanModulesFolders(mockModules, ModulesTestPath); err != nil { + t.Errorf("failed to clean up the downloaded modules, %s", err.Error()) + } + + folders, err = getFolderAsMap(ModulesTestPath) + if err != nil { + t.Errorf(err.Error()) + } + + if _, exist := folders[TerraCrustModuleName]; exist { + t.Errorf("cleanup-failed terracrust folder still exists") + } + + if _, exist := folders[ZonesModuleName]; exist { + t.Errorf("cleanup-failed zones folder still exists") + } + + if _, exist := folders[NamingModuleName]; exist { + t.Errorf("cleanup-failed naming folder still exists") + } + + if len(folders) != 0 { + t.Errorf("Expected 0 folder count received %d", len(folders)) + } +} + +func TestFailBadUrl(t *testing.T) { + t.Parallel() + log := logger.NewSimple() + gitDriver := version_control.InitGitProvider(log) + + err := gitDriver.CloneModules(mockBadModules, ModulesTestPath) + if err == nil { + t.Errorf("expected error received error nil") + } +} + +func getFolderAsMap(path string) (map[string]struct{}, error) { + folders := make(map[string]struct{}) + + files, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + for _, file := range files { + if file.IsDir() { + folders[file.Name()] = struct{}{} + } + } + + return folders, err +} diff --git a/internal/services/template_api.go b/internal/services/template_api.go index 0d742ae..99bc5d4 100644 --- a/internal/services/template_api.go +++ b/internal/services/template_api.go @@ -32,6 +32,7 @@ func NewTemplateAPI() *TemplateAPI { "SimpleWrap": SimpleWrap, "ModuleDataWrapper": ModuleDataWrapper, "GetDefaults": GetDefaults, + "GetRequired": GetRequired, }, } } @@ -62,3 +63,12 @@ func GetDefaults(moduleName string, modulesMap *templates.MainModuleTF) string { return sb.String() } + +func GetRequired(moduleName string, modulesMap *templates.MainModuleTF) string { + var sb strings.Builder + for k := range modulesMap.Module[moduleName].RequiredFields { + sb.WriteString(fmt.Sprintf(mainRequiredVarRowTemplate, k)) + } + + return sb.String() +} diff --git a/internal/services/templates/variables_modules_tf.go b/internal/services/templates/variables_modules_tf.go index 73f5e76..1cf342d 100644 --- a/internal/services/templates/variables_modules_tf.go +++ b/internal/services/templates/variables_modules_tf.go @@ -14,7 +14,7 @@ package templates -type VariblesModuleList map[string]*VariablesModulesTF +type VariablesModuleList map[string]*VariablesModulesTF type VariablesModulesTF struct { ModuleName string diff --git a/internal/services/terraform.go b/internal/services/terraform.go index 89b9574..c235ac6 100644 --- a/internal/services/terraform.go +++ b/internal/services/terraform.go @@ -27,9 +27,10 @@ const ( moduleDescription = `<