From 0e175828318ed4a2538dc0ad1ab4b9f78dcd272c Mon Sep 17 00:00:00 2001 From: "Schroeter Dominik; BMW Group" Date: Tue, 27 Nov 2018 07:33:30 +0100 Subject: [PATCH] Add credential management (decrypt and apply) This enables migration of credentials between Jenkins instances. Butler will get the XML config of any folder and will ingest the encrypted secrets. The encrypted secrets are then decrypted via a Groovy script executed on Jenkins master. This needs most of the times admin privileges. In a second step you can pipe in the decrypted credentials and apply them to another folder or even instance. Also a Groovy script is leveraged in order to achieve this. --- README.md | 17 +++++-- credentials.go | 32 ++++++++++++ folder.go | 82 +++++++++++++++++++++++++++++- folder_test.go | 64 +++++++++++++++++++++++- groovy_scripts.go | 125 ++++++++++++++++++++++++++++++++++++++++++++++ main.go | 92 ++++++++++++++++++++++++++++++++++ 6 files changed, 406 insertions(+), 6 deletions(-) create mode 100644 credentials.go create mode 100644 groovy_scripts.go diff --git a/README.md b/README.md index 61f7100..ed3106b 100644 --- a/README.md +++ b/README.md @@ -64,23 +64,32 @@ In order to always skip folders, you may set the environment variable `JENKINS_S ### Jobs Management ``` -$ butler jobs export --server localhost:8080 --username admin --password admin --skip-folder +$ butler jobs export --server localhost:8080 --skip-folder ``` ``` -$ butler jobs import --server localhost:8080 --username admin --password admin +$ butler jobs import --server localhost:8080 ``` ### Plugins Management ``` -$ butler plugins export --server localhost:8080 --username admin --password admin +$ butler plugins export --server localhost:8080 ``` ``` -$ butler plugins import --server localhost:8080 --username admin --password admin +$ butler plugins import --server localhost:8080 ``` +### Credentials Management + +``` +$ butler credentials decrypt --server localhost:8080 --folder foo/bar > decryptedCredentials.json +``` + +``` +$ cat decryptedCredentials.json | butler credentials apply --server localhost:8080 --folder bar/foo +``` ## Tutorials * [Butler CLI: Import/Export Jenkins Plugins & Jobs](http://www.blog.labouardy.com/butler-cli-import-export-jenkins-plugins-jobs/) diff --git a/credentials.go b/credentials.go new file mode 100644 index 0000000..24a2cbc --- /dev/null +++ b/credentials.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" +) + +func DecryptFolderCredentials(url string, folderName string, username string, password string) error { + folder, _ := GetFolder(url, folderName, username, password) + credentials := folder.GetCredentials() + script := GetDecryptScriptForCredentials(credentials) + response := ExecuteGroovyScriptOnJenkins(script, url, username, password) + fmt.Println(response) + return nil +} + +func ApplyFolderCredentials(url string, folderName string, username string, password string) error { + var credentials Credentials + + err := json.NewDecoder(os.Stdin).Decode(&credentials) + if err != nil { + log.Fatal(err) + panic(err) + } + script := GetApplyScriptForCredentials(credentials, folderName) + response := ExecuteGroovyScriptOnJenkins(script, url, username, password) + fmt.Println(response) + + return nil +} diff --git a/folder.go b/folder.go index b3af929..39e00c2 100644 --- a/folder.go +++ b/folder.go @@ -1,6 +1,47 @@ package main -import "strings" +import ( + "encoding/xml" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" +) + +type JenkinsFolder struct { + Properties struct { + CredentialProperty struct { + DomainCredentials struct { + Class string `xml:"class,attr"` + Entry struct { + Credentials Credentials `xml:"java.util.concurrent.CopyOnWriteArrayList"` + } `xml:"entry"` + } `xml:"domainCredentialsMap"` + } `xml:"com.cloudbees.hudson.plugins.folder.properties.FolderCredentialsProvider_-FolderCredentialsProperty"` + } `xml:"properties"` +} + +type Credentials struct { + UsernamePassword []UsernamePasswordCredential `xml:"com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl" json:"userpass"` + SecretFile []SecretFileCredential `xml:"org.jenkinsci.plugins.plaincredentials.impl.FileCredentialsImpl" json:"secretfile"` +} + +type UsernamePasswordCredential struct { + Plugin string `xml:"plugin,attr" json:"plugin"` + ID string `xml:"id" json:"id"` + Description string `xml:"description" json:"description"` + Username string `xml:"username" json:"username"` + Password string `xml:"password" json:"password"` +} + +type SecretFileCredential struct { + Plugin string `xml:"plugin,attr"` + ID string `xml:"id" json:"id"` + Description string `xml:"description" json:"description"` + FileName string `xml:"fileName" json:"fileName"` + SecretBytes string `xml:"secretBytes" json:"secretBytes"` +} func GetFolderURL(url string, folderName string) string { if folderName == "" { @@ -15,3 +56,42 @@ func GetFolderURL(url string, folderName string) string { path := strings.Replace(folderName, "/", "/job/", -1) return url + path } + +func parseJenkinsFolder(xmlInput []byte) JenkinsFolder { + var folder JenkinsFolder + xmlInput = []byte(strings.Trim(string(xmlInput), "")) + xml.Unmarshal(xmlInput, &folder) + return folder +} + +func (folder *JenkinsFolder) GetCredentials() Credentials { + return folder.Properties.CredentialProperty.DomainCredentials.Entry.Credentials +} + +func GetFolder(url string, folderName string, username string, password string) (JenkinsFolder, error) { + url = fmt.Sprintf("%s/config.xml", GetFolderURL(url, folderName)) + + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + req.SetBasicAuth(username, password) + if err != nil { + return JenkinsFolder{}, err + } + + resp, err := client.Do(req) + if err != nil { + return JenkinsFolder{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == 401 { + return JenkinsFolder{}, errors.New("Unauthorized 401") + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return JenkinsFolder{}, err + } + + return parseJenkinsFolder(data), nil +} diff --git a/folder_test.go b/folder_test.go index 09c80a7..cc332ba 100644 --- a/folder_test.go +++ b/folder_test.go @@ -1,6 +1,10 @@ package main -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestGetFolderURL(t *testing.T) { type args struct { @@ -69,3 +73,61 @@ func TestGetFolderURL(t *testing.T) { }) } } + +func TestParseJenkinsFolder(t *testing.T) { + assert := assert.New(t) + xml := ` + + + Ansible + + + + + + + + + + test + Test + Test + {AQAAABAAAAAQ7EzV5N/fXZEKM9HyG+1T66P67iqU+tptVCNuvNX1TM0=} + + + deploy-key-file + blub + accessKeys.csv + {bEtRRJ+hCoQHgEAmcGhAOlKFx6J5tVuKmwdBVSgdq4zkktsLwG1zHO6swI3mQ5z9UhbgRRHDf2W8oSHlfmno8+KHWKWKyNmQUL5cv6/8n5JnmvsMGx+DT4KJL2XDVl33nuNbDpkcJEDGBWqb2hA47iRtW6h4mxlbNja5E12eUMs=} + + + + + + + + + + + all + false + false + + + + all + + + + + false + + + +` + + got := parseJenkinsFolder([]byte(xml)) + + assert.NotNil(got, "Should not be nil.") + assert.Equal(got.GetCredentials().UsernamePassword[0].ID, "test") +} diff --git a/groovy_scripts.go b/groovy_scripts.go new file mode 100644 index 0000000..6903318 --- /dev/null +++ b/groovy_scripts.go @@ -0,0 +1,125 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +func GetDecryptScriptForCredentials(credentials Credentials) string { + marshalledCredentials, _ := json.Marshal(credentials) + return strings.Replace(decryptScriptTemplate, "<>", string(marshalledCredentials), 1) +} + +const decryptScriptTemplate = `import groovy.json.JsonSlurperClassic +import groovy.json.JsonOutput + + +def json = """<>""" + +def data = new JsonSlurperClassic().parseText(json) +data.userpass.each { + it.password = hudson.util.Secret.fromString(it.password).getPlainText() +} + +data.secretfile.each { + it.secretBytes = new String(com.cloudbees.plugins.credentials.SecretBytes.fromString(it.secretBytes).getPlainData(), "ASCII") +} + +println JsonOutput.toJson(data)` + +func GetApplyScriptForCredentials(credentials Credentials, folderPath string) string { + marshalledCredentials, _ := json.Marshal(credentials) + templated := strings.Replace(createOrUpdateCredentialsTemplate, "<>", string(marshalledCredentials), 1) + templated = strings.Replace(templated, "<>", folderPath, 1) + return templated +} + +const createOrUpdateCredentialsTemplate = `import com.cloudbees.hudson.plugins.folder.properties.FolderCredentialsProvider.FolderCredentialsProperty +import com.cloudbees.hudson.plugins.folder.AbstractFolder +import com.cloudbees.hudson.plugins.folder.Folder +import jenkins.model.* +import com.cloudbees.plugins.credentials.* +import com.cloudbees.plugins.credentials.common.* +import com.cloudbees.plugins.credentials.domains.* +import com.cloudbees.plugins.credentials.impl.* +import org.jenkinsci.plugins.plaincredentials.* +import org.jenkinsci.plugins.plaincredentials.impl.* +import org.apache.commons.fileupload.FileItem +import groovy.json.JsonSlurperClassic +import groovy.json.JsonOutput + +def createOrUpdateCredential(credentialStore, newCredential, existingCredentials) { + def existingCredential = existingCredentials.find{c -> c.getId() == newCredential.getId()} + if (existingCredential) + credentialStore.updateCredentials(Domain.global(), existingCredential, newCredential) + else + credentialStore.addCredentials(Domain.global(), newCredential) +} + +def json = """<>""" +String folderPath = "<>" +def data = new JsonSlurperClassic().parseText(json) + +Jenkins.instance.getAllItems(Folder.class) + .findAll{it.fullName.equals(folderPath)} + .each{ + AbstractFolder folderAbs = AbstractFolder.class.cast(it) + println "We're at ${folderAbs.fullName}" + FolderCredentialsProperty property = folderAbs.getProperties().get(FolderCredentialsProperty.class) + if(property == null){ + property = new FolderCredentialsProperty() + folderAbs.addProperty(property) + } + + def store = property.getStore() + def existingCredentials = property.getCredentials() + data.userpass.each { + Credentials c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, it.id, it.description, it.username, it.password) + createOrUpdateCredential(store, c, existingCredentials) + } + data.secretfile.each { + def rawSecretFile = it + fileItem = [ getName: { return rawSecretFile.fileName}, get: { return rawSecretFile.secretBytes.getBytes() } ] as FileItem + secretFile = new FileCredentialsImpl( + CredentialsScope.GLOBAL, + rawSecretFile.id, + rawSecretFile.description, + fileItem, // Don't use FileItem + null, + "") + createOrUpdateCredential(store, secretFile, existingCredentials) + } + println existingCredentials.toString() +}` + +func ExecuteGroovyScriptOnJenkins(script string, rawUrl string, username string, password string) string { + apiURL := fmt.Sprintf("%s/scriptText", rawUrl) + data := url.Values{} + data.Set("script", script) + body := strings.NewReader(data.Encode()) + req, err := http.NewRequest("POST", apiURL, body) + if err != nil { + panic(err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + crumb, err := GetCrumb(rawUrl, username, password) + if err != nil { + fmt.Errorf("No crumb issueing possible: %v", err) + } else { + req.Header.Set(crumb[0], crumb[1]) + } + + req.SetBasicAuth(username, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + responseBody, _ := ioutil.ReadAll(resp.Body) + return string(responseBody) +} diff --git a/main.go b/main.go index e11252e..ee96e41 100644 --- a/main.go +++ b/main.go @@ -123,6 +123,98 @@ func main() { }, }, }, + { + Name: "credentials", + Usage: "Jenkins Credential Management", + Subcommands: []cli.Command{ + { + Name: "decrypt", + Usage: "Decrypt credentials of Jenkins folder", + Aliases: []string{"d"}, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "server", + Usage: "Jenkins url", + }, + cli.StringFlag{ + Name: "folder, f", + Usage: "Jenkins Folder", + }, + cli.StringFlag{ + Name: "username, u", + Usage: "Jenkins username", + EnvVar: "JENKINS_USER", + }, + cli.StringFlag{ + Name: "password, p", + Usage: "Jenkins password", + EnvVar: "JENKINS_PASSWORD", + }, + }, + Action: func(c *cli.Context) error { + var url = getSanitizedUrl(c.String("server")) + var username = c.String("username") + var password = c.String("password") + var folder = c.String("folder") + + if url == "" || folder == "" { + cli.ShowSubcommandHelp(c) + return nil + } + + err := DecryptFolderCredentials(url, folder, username, password) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + return nil + }, + }, + { + Name: "apply", + Usage: "Apply (from STDIN) credentials of Jenkins folder", + Aliases: []string{"a"}, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "server", + Usage: "Jenkins url", + }, + cli.StringFlag{ + Name: "folder, f", + Usage: "Jenkins Folder", + }, + cli.StringFlag{ + Name: "username, u", + Usage: "Jenkins username", + EnvVar: "JENKINS_USER", + }, + cli.StringFlag{ + Name: "password, p", + Usage: "Jenkins password", + EnvVar: "JENKINS_PASSWORD", + }, + }, + Action: func(c *cli.Context) error { + var url = getSanitizedUrl(c.String("server")) + var username = c.String("username") + var password = c.String("password") + var folder = c.String("folder") + + if url == "" || folder == "" { + cli.ShowSubcommandHelp(c) + return nil + } + + err := ApplyFolderCredentials(url, folder, username, password) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + return nil + }, + }, + }, + }, { Name: "plugins", Usage: "Jenkins Plugins Management",