From 882ec35a8359ada38456d39ce10aefb5da178d6b Mon Sep 17 00:00:00 2001 From: Aashirwad Jain Date: Tue, 26 Nov 2024 10:15:52 +0530 Subject: [PATCH] feat(bulk_user_import): automates import of existing users, groups as terraform resources (#2778) --- examples/bulk_users_import/README.md | 29 +++ .../bulk_users_import_constants.go | 67 ++++++ .../bulk_users_import_helpers.go | 17 ++ .../bulk_users_import_types.go | 30 +++ examples/bulk_users_import/go.mod | 25 +++ examples/bulk_users_import/go.sum | 52 +++++ examples/bulk_users_import/main.go | 207 ++++++++++++++++++ .../run_bulk_users_import.sh | 10 + 8 files changed, 437 insertions(+) create mode 100644 examples/bulk_users_import/README.md create mode 100644 examples/bulk_users_import/bulk_users_import_constants.go create mode 100644 examples/bulk_users_import/bulk_users_import_helpers.go create mode 100644 examples/bulk_users_import/bulk_users_import_types.go create mode 100644 examples/bulk_users_import/go.mod create mode 100644 examples/bulk_users_import/go.sum create mode 100644 examples/bulk_users_import/main.go create mode 100644 examples/bulk_users_import/run_bulk_users_import.sh diff --git a/examples/bulk_users_import/README.md b/examples/bulk_users_import/README.md new file mode 100644 index 000000000..ce20903bc --- /dev/null +++ b/examples/bulk_users_import/README.md @@ -0,0 +1,29 @@ +### Overview +The `run_bulk_users_import.sh` script is a Bash script used to run a Go program that imports users in bulk (belonging to the specified group) into your Terraform configuration, to be saved to the state as `newrelic_user` resources (and `newrelic_group`, for the group with the ID specified), so as to facilitate controlling the imported user and group via Terraform, using resources in the New Relic Terraform Provider. + +This is specifically useful in cases where a huge number of users were created in the New Relic One UI, added to a group, and would now like to be controlled via the `newrelic_user` and `newrelic_group` resources respectively, along with future users who would be added to the group via these resources in the New Relic Terraform Provider. + + The script works as follows - +- Fetch users from the group with the ID specified, +- Get details of all of such users, in alignment with expected arguments of the `newrelic_user` resource, +- Write the attributes and values of each user (and the group specified) to strings in HCL (Terraform format), +- Write the generated Terraform configuration to `generated.tf` in the filepath specified, and +- Run a `terraform import` command in the filepath specified to also import all of these users (and the group) to the current Terraform state. + + +#### Arguments +--groupId : The ID of the group to which the users will be imported. This is a required flag. +--apiKey : The User API key used for authentication. This is an optional flag, and can only be skipped if your environment has a `NEW_RELIC_API_KEY` that can be defaulted to. +--filePath : The path to the file containing the user data to be imported. This is a required flag. + +#### Example Usage +```sh +bash run_bulk_users_import.sh --groupId XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX --filePath ../../testing +``` + +```sh +bash run_bulk_users_import.sh --groupId XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX --apiKey XXXX-XXXXXXXXXXXXXXXX --filePath ../../testing +``` + + + diff --git a/examples/bulk_users_import/bulk_users_import_constants.go b/examples/bulk_users_import/bulk_users_import_constants.go new file mode 100644 index 000000000..1dec46237 --- /dev/null +++ b/examples/bulk_users_import/bulk_users_import_constants.go @@ -0,0 +1,67 @@ +package main + +const GET_GROUPS_AND_USERS_QUERY = ` +query( + $groupId: [ID!] +){ + actor { + organization { + userManagement { + authenticationDomains { + nextCursor + authenticationDomains { + name + id + groups(id: $groupId) { + groups { + id + displayName + users { + users { + email + id + name + timeZone + } + } + } + } + } + } + } + } + } +} +` + +const GET_USER_DETAILS_QUERY = ` +query( + $userId: [ID!] +){ + actor { + organization { + userManagement { + authenticationDomains { + nextCursor + authenticationDomains { + name + id + users(id: $userId) { + users { + email + id + name + type { + displayName + } + } + } + } + } + } + } + } +}` + +const NERDGRAPH_API_ENDPOINT = "https://api.newrelic.com/graphql" +const NERDGRAPH_API_ENDPOINT_EU = "https://api.newrelic.com/graphql" diff --git a/examples/bulk_users_import/bulk_users_import_helpers.go b/examples/bulk_users_import/bulk_users_import_helpers.go new file mode 100644 index 000000000..c7df21b6b --- /dev/null +++ b/examples/bulk_users_import/bulk_users_import_helpers.go @@ -0,0 +1,17 @@ +package main + +import ( + "context" + + "github.com/machinebox/graphql" +) + +func RunGraphQLRequest(client *graphql.Client, query string, variables map[string]interface{}, apiKey string, response interface{}) error { + request := graphql.NewRequest(query) + for key, value := range variables { + request.Var(key, value) + } + request.Header.Set("Api-Key", apiKey) + + return client.Run(context.Background(), request, response) +} diff --git a/examples/bulk_users_import/bulk_users_import_types.go b/examples/bulk_users_import/bulk_users_import_types.go new file mode 100644 index 000000000..3b5043079 --- /dev/null +++ b/examples/bulk_users_import/bulk_users_import_types.go @@ -0,0 +1,30 @@ +package main + +import ( + "github.com/newrelic/newrelic-client-go/v2/pkg/usermanagement" +) + +type authenticationDomainsResponse struct { + Actor usermanagement.Actor `json:"actor"` +} + +type ResourceUser struct { + id string + name string + email_id string + authentication_domain_id string + user_type string +} + +type ResourceGroup struct { + id string + name string + authentication_domain_id string + user_ids []string +} + +var userTier = map[string]string{ + "Basic": "BASIC_USER_TIER", + "Core": "CORE_USER_TIER", + "Full platform": "FULL_USER_TIER", +} diff --git a/examples/bulk_users_import/go.mod b/examples/bulk_users_import/go.mod new file mode 100644 index 000000000..f60f386a2 --- /dev/null +++ b/examples/bulk_users_import/go.mod @@ -0,0 +1,25 @@ +module bulk_users_import + +go 1.22.6 + +require ( + github.com/machinebox/graphql v0.2.2 + github.com/newrelic/newrelic-client-go/v2 v2.51.3 + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/matryer/is v1.4.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.9.0 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + github.com/valyala/fastjson v1.6.4 // indirect + golang.org/x/sys v0.20.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/bulk_users_import/go.sum b/examples/bulk_users_import/go.sum new file mode 100644 index 000000000..29b65f2ce --- /dev/null +++ b/examples/bulk_users_import/go.sum @@ -0,0 +1,52 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= +github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= +github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/newrelic/newrelic-client-go/v2 v2.51.3 h1:Bu/cUs6nfMjQMPBcxxHt4Xm30tKDT7ttYy/XRDsWP6Y= +github.com/newrelic/newrelic-client-go/v2 v2.51.3/go.mod h1:+RRjI3nDGWT3kLm9Oi3QxpBm70uu8q1upEHBVWCZFpo= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/bulk_users_import/main.go b/examples/bulk_users_import/main.go new file mode 100644 index 000000000..e284e9bea --- /dev/null +++ b/examples/bulk_users_import/main.go @@ -0,0 +1,207 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "strings" + + "github.com/machinebox/graphql" + "golang.org/x/exp/maps" +) + +func main() { + groupId, apiKey, filePath := parseFlags() + + client := graphql.NewClient(NERDGRAPH_API_ENDPOINT) + + logInfo("main", "Fetching group and users...") + resourceGroup, resourceUserById := fetchGroupAndUsers(client, groupId, apiKey) + + logInfo("main", "Updating user details...") + updateUserDetails(client, apiKey, resourceUserById) + + logInfo("main", "Generating Terraform files...") + fileContent, importCommand := generateTerraformFiles(resourceGroup, resourceUserById) + + logInfo("main", "Creating Terraform file...") + createTerraformFile(fileContent, filePath) + + logInfo("main", "Importing Terraform resources...") + importTerraformResources(importCommand, filePath) +} + +func parseFlags() (string, string, string) { + logInfo("parseFlags", "Parsing command line arguments...") + groupId := flag.String("groupId", "", "The group ID") + apiKey := flag.String("apiKey", os.Getenv("NEW_RELIC_API_KEY"), "The API key") + filePath := flag.String("filePath", "", "The file path") + + flag.Parse() + + if *groupId == "" || *filePath == "" { + log.Fatalf("[ERROR] parseFlags: groupId and filePath are required arguments") + } + + logInfo("parseFlags", "Finished parsing command line arguments.") + return *groupId, *apiKey, *filePath +} + +func fetchGroupAndUsers(client *graphql.Client, groupId string, apiKey string) (ResourceGroup, map[string]ResourceUser) { + logInfo("fetchGroupAndUsers", "Starting to fetch group and users...") + query := GET_GROUPS_AND_USERS_QUERY + response := authenticationDomainsResponse{} + err := RunGraphQLRequest(client, query, map[string]interface{}{"groupId": groupId}, apiKey, &response) + if err != nil { + log.Fatalf("[ERROR] fetchGroupAndUsers: Error fetching group and users: %v", err) + } + + var resourceGroup ResourceGroup + resourceUserById := make(map[string]ResourceUser) + + if len(response.Actor.Organization.UserManagement.AuthenticationDomains.AuthenticationDomains) > 0 { + authDomains := response.Actor.Organization.UserManagement.AuthenticationDomains.AuthenticationDomains + for _, authDomain := range authDomains { + if len(authDomain.Groups.Groups) > 0 { + for _, group := range authDomain.Groups.Groups { + resourceGroup.id = group.ID + resourceGroup.name = group.DisplayName + resourceGroup.authentication_domain_id = authDomain.ID + + for _, user := range group.Users.Users { + resourceUser := ResourceUser{ + id: user.ID, + name: user.Name, + email_id: user.Email, + authentication_domain_id: authDomain.ID, + } + resourceUserById[user.ID] = resourceUser + resourceGroup.user_ids = append(resourceGroup.user_ids, user.ID) + } + } + } + } + } + + logInfo("fetchGroupAndUsers", "Finished fetching group and users.") + return resourceGroup, resourceUserById +} + +func updateUserDetails(client *graphql.Client, apiKey string, resourceUserById map[string]ResourceUser) { + logInfo("updateUserDetails", "Starting to update user details...") + query := GET_USER_DETAILS_QUERY + response := authenticationDomainsResponse{} + err := RunGraphQLRequest(client, query, map[string]interface{}{"userId": maps.Keys(resourceUserById)}, apiKey, &response) + if err != nil { + log.Fatalf("[ERROR] updateUserDetails: Error updating user details: %v", err) + } + + if len(response.Actor.Organization.UserManagement.AuthenticationDomains.AuthenticationDomains) > 0 { + authDomains := response.Actor.Organization.UserManagement.AuthenticationDomains.AuthenticationDomains + for _, authDomain := range authDomains { + if len(authDomain.Users.Users) > 0 { + for _, user := range authDomain.Users.Users { + val, ok := resourceUserById[user.ID] + if ok { + val.user_type = userTier[user.Type.DisplayName] + resourceUserById[user.ID] = val + } + } + } + } + } + + logInfo("updateUserDetails", "Finished updating user details.") +} + +func generateTerraformFiles(resourceGroup ResourceGroup, resourceUserById map[string]ResourceUser) (string, string) { + logInfo("generateTerraformFiles", "Starting to generate Terraform files...") + var fileContent string + + groupStr := `resource "newrelic_group" "%s" { + name = "%s" + authentication_domain_id = "%s" + user_ids = %s + }` + + var userIdList string = "[" + + resourceName := resourceGroup.name + resourceName = strings.ReplaceAll(resourceName, " ", "-") + + importCommand := "terraform import newrelic_group." + resourceName + " " + resourceGroup.id + " && " + + for _, user := range resourceUserById { + userStr := `resource "newrelic_user" "%s" { + name = "%s" + email_id = "%s" + authentication_domain_id = "%s" + user_type = "%s" + }` + + importCommand += "terraform import newrelic_user." + user.name + " " + user.id + " && " + userIdList += ("newrelic_user." + user.name + ".id,") + + resourceName := user.name + resourceName = strings.ReplaceAll(resourceName, " ", "-") + + fileContent += fmt.Sprintf(userStr, resourceName, user.name, user.email_id, user.authentication_domain_id, user.user_type) + fileContent += "\n\n" + } + + importCommand = importCommand[:len(importCommand)-4] + userIdList = userIdList[:len(userIdList)-1] + userIdList += "]" + + importCommand += " && terraform fmt " + + fileContent += fmt.Sprintf(groupStr, resourceName, resourceGroup.name, resourceGroup.authentication_domain_id, userIdList) + fileContent += "\n\n" + + logInfo("generateTerraformFiles", "Finished generating Terraform files.") + return fileContent, importCommand +} + +func createTerraformFile(content string, path string) { + logInfo("createTerraformFile", "Starting to create Terraform file...") + filePath := path + "/generated.tf" + + if _, err := os.Stat(filePath); err == nil { + log.Printf("[ERROR] createTerraformFile: File already exists at path: %s", filePath) + return + } + + file, err := os.Create(filePath) + if err != nil { + log.Fatalf("[ERROR] createTerraformFile: Error while creating file: %v", err) + } + + defer file.Close() + + _, err = io.WriteString(file, content) + if err != nil { + log.Fatalf("[ERROR] createTerraformFile: Error while writing file: %v", err) + } + + logInfo("createTerraformFile", "Finished creating Terraform file.") +} + +func importTerraformResources(command string, path string) { + logInfo("importTerraformResources", "Starting to import Terraform resources...") + cmd := exec.Command("bash", "-c", "cd "+path+" && "+command) + + output, err := cmd.CombinedOutput() + if err != nil { + log.Fatalf("[ERROR] importTerraformResources: Error executing command: %v", err) + } + + log.Printf("[INFO] importTerraformResources: Output:\n%s\n", output) + logInfo("importTerraformResources", "Finished importing Terraform resources.") +} + +func logInfo(functionName, message string) { + log.Printf("[INFO] %s: %s", functionName, message) +} diff --git a/examples/bulk_users_import/run_bulk_users_import.sh b/examples/bulk_users_import/run_bulk_users_import.sh new file mode 100644 index 000000000..65d785cbc --- /dev/null +++ b/examples/bulk_users_import/run_bulk_users_import.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ -z "$1" ] || [ -z "$3" ]; then + echo "This command requires the flags --groupId and --apiKey to be provided, with valid values." + echo "Usage: $0 --groupId --apiKey --filePath (or) --groupId --filePath " + exit 1 +fi + +go mod tidy +go run . "$@" \ No newline at end of file