diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b03a4d --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +JIRA Replicator +=============== + +A library (and tool) to automatically back-up your JIRA instance. + +Especially handy for JIRA Cloud users. + + +Tool Usage +---------- + +### Configuration + +Configuration is done using the following environment variables. + +* `JIRA_URL` +* `JIRA_USERNAME` +* `JIRA_PASSWORD` + +### Usage + +Generate the backup using: + + $ jira-backup backup + +Download the backup using: + + $ jira-backup download -o FILE_PATH + +Copy the backup to S3: + + $ jira-backup s3 + +Run it on a loop to make a backup every 48 hours: + + $ jira-backup daemon + +Please see `jira-backup help s3` for full docs on all available options and how to set defaults. + +Backups on JIRA cloud are limited to once per 48 hours (after completion of previous backup). +If you exceed this you will be notified of when you can re-try the request. + +### Library Usage + +See godoc. \ No newline at end of file diff --git a/client/backup.go b/client/backup.go new file mode 100644 index 0000000..cc9baec --- /dev/null +++ b/client/backup.go @@ -0,0 +1,49 @@ +package client + +import ( + "time" + "fmt" + "regexp" + "strings" +) + +// BackupRateExceeded is an error which means that the +// rate limit on the backup API has been exceeded. +type BackupRateExceeded struct { + retryAt time.Time +} + +func (e BackupRateExceeded) Error() string { + return fmt.Sprintf("Backup rate exceeded. Retry in %s", e.RetryIn()) +} + + +func (e BackupRateExceeded) RetryIn() time.Duration { + return e.retryAt.Sub(time.Now()) +} + +func (e BackupRateExceeded) RetryAt() time.Time { + return e.retryAt +} + +// FromResponse creates a BackupRateExceeded from the given text. +// It will attempt to extract the retry time from the text. +func (BackupRateExceeded) FromResponse(text string) BackupRateExceeded { + + // Helpfully Atlassian gives the response time in a format easily passed + // to time.ParseDuration + exp := regexp.MustCompile("((\\d+)h)?((\\d+)m)?") + + matches := exp.FindAllString(text, -1) + durationStr := strings.Join(matches, "") + + dur, err := time.ParseDuration(durationStr) + + if err == nil { + return BackupRateExceeded { + retryAt: time.Now().Round(time.Minute).Add(dur), + } + } + + return BackupRateExceeded{retryAt: time.Now()} +} \ No newline at end of file diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..56891ce --- /dev/null +++ b/client/client.go @@ -0,0 +1,223 @@ +package client + +import ( + "net/url" + "net/http" + "log" + "os" + "io/ioutil" + "encoding/json" + "errors" + "bytes" + + "io" + "time" + "strconv" +) + +// Client represents a JIRA backup client +type Client struct { + BaseURL url.URL + HTTP *http.Client + Log *log.Logger + + username string + password string +} + +type BackupStatus struct { + Status string `json:"status"` + Progress int `json:"progress"` + DownloadPath string `json:"result"` +} + + +// New creates a new JIRA Backup Client +func New(baseURL *url.URL, username, password string) *Client { + + return &Client{ + BaseURL: *baseURL, + HTTP: &http.Client{Transport: http.DefaultTransport}, + Log: log.New(os.Stdout, "jira-client ", log.LstdFlags), + + username: username, + password: password, + } +} + +func (c Client) makeRequest(req *http.Request) (*http.Response, error) { + + if req.Header == nil { + req.Header = http.Header{} + } + + req.Header.Set("User-Agent", "Jira Replicator +https://bitbucket.org/mr-zen/jira-replicator") + req.SetBasicAuth(c.username, c.password) + + st := time.Now() + res, err := c.HTTP.Do(req) + dt := time.Now().Sub(st) + + if err == nil { + c.Log.Println(req.Method, req.URL, res.StatusCode, res.Header.Get("Content-Length"), dt) + } + + return res, err +} + +// CreateBackup creates a new JIRA backup. +func (c Client) CreateBackup(includeAttachments bool) error { + + u := c.BaseURL + u.Path = "/rest/backup/1/export/runbackup" + + req := &http.Request{ + Method: http.MethodPost, + URL: &u, + Header: http.Header{ + "Accept": []string{"application/json"}, + "Content-Type": []string{"application/json"}, + }, + } + + params := make(map[string]bool) + params["cbAttachments"] = includeAttachments + body, _ := json.Marshal(params) + + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + + res, err := c.makeRequest(req) + + if err != nil { + return err + } + + if res.StatusCode >= 400 { + if res.StatusCode == http.StatusPreconditionFailed /* 412 */ { + // We got us some json, let's get some deets. + body, err := ioutil.ReadAll(res.Body) + + if err != nil { + return err + } + + respData := make(map[string]string) + + err = json.Unmarshal(body, &respData) + + if err != nil { + return err + } + + return BackupRateExceeded{}.FromResponse(respData["error"]) + } + + content, _ := ioutil.ReadAll(res.Body) + + return errors.New(string(content)) + } + + + + return err +} + +// Download backup downloads the current JIRA backup +func (c Client) DownloadBackup() (io.ReadCloser, int, error) { + + // Check the backup is ready and waiting. + status, err := c.GetBackupStatus() + + if err != nil { + return nil, 0, err + } + + if status.Status != "Success" { + return nil, 0, errors.New(status.Status) + } + + downloadURL, err := url.Parse(status.DownloadPath) + + u := c.BaseURL + u.Path = "/plugins/servlet/"+downloadURL.Path + u.RawQuery = downloadURL.Query().Encode() + + req := &http.Request{ + Method: http.MethodGet, + URL: &u, + } + + res, err := c.makeRequest(req) + + if err != nil { + return nil, 0, err + } + + length, _ := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) + + + return res.Body, int(length), nil +} + +func (c Client) GetBackupStatus() (BackupStatus, error) { + + u := c.BaseURL + u.Path = "/rest/backup/1/export/lastTaskId" + q := u.Query() + q.Set("_", strconv.Itoa(int(time.Now().Unix()))) + u.RawQuery = q.Encode() + + // First determine the task ID. + req := &http.Request{ + Method: http.MethodGet, + URL: &u, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + } + + res, err := c.makeRequest(req) + + if err != nil { + return BackupStatus{}, err + } + + if res.StatusCode >= 400 { + return BackupStatus{}, errors.New(res.Status) + } + + body, _ := ioutil.ReadAll(res.Body) + taskId := string(body) + + // Get the status + u = c.BaseURL + u.Path = "/rest/backup/1/export/getProgress" + q = u.Query() + q.Set("taskId", taskId) + q.Set("_", strconv.Itoa(int(time.Now().Unix()))) + u.RawQuery = q.Encode() + + req = &http.Request{ + Method: http.MethodGet, + URL: &u, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + } + + res, err = c.makeRequest(req) + + if err != nil { + return BackupStatus{}, err + } + + body, _ = ioutil.ReadAll(res.Body) + + var b BackupStatus + err = json.Unmarshal(body, &b) + + return b, err + + return b, err + +} \ No newline at end of file diff --git a/jira.toml b/jira.toml new file mode 100644 index 0000000..54952bc --- /dev/null +++ b/jira.toml @@ -0,0 +1,7 @@ +[s3] +region = "eu-west-1" +bucket = "YOUR_JIRA_BACKUP_BUCKET" +storage_class = "STANDARD_IA" # Recommended for backups. + +[kms] +key = "arn:aws:kms:eu-west-1:AWS_ACCOUNT_ID:key/KEY_UUID" # Set this to enable SSE. \ No newline at end of file diff --git a/replicator.go b/replicator.go new file mode 100644 index 0000000..df88e16 --- /dev/null +++ b/replicator.go @@ -0,0 +1,353 @@ +package main + +import ( + "github.com/urfave/cli" + "github.com/spf13/viper" + "os" + "fmt" + "net/url" + "github.com/mrzen/jira-replicator/client" + "errors" + "io" + "github.com/cheggaaa/pb" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "time" + "bytes" + "log" + "math" +) + +const partSize = 64 * 1<<20 + +func main() { + + app := cli.NewApp() + app.Name = "JIRA Replicator" + + app.Before = setup + + + app.Commands = []cli.Command{ + { + Name: "backup", + Aliases: []string{"b"}, + Description: "Create a new backup", + Action: makeBackup, + }, + + { + Name: "download", + Aliases: []string{"d"}, + Description: "Download backup file", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "o", + Usage: "Output file location", + + }, + }, + Action: downloadBackup, + }, + + { + Name: "s3", + Description: "Upload backup to S3", + Action: backupToS3, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config", + Usage: "Configuration file path", + }, + }, + }, + + { + Name: "daemon", + Description: "Replication daemon, creates backups and copies to S3 as often as possible.", + Action: replicate, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config", + Usage: "Configuration file path", + }, + }, + }, + } + + app.Run(os.Args) + +} + +func setup(ctx *cli.Context) error { + viper.SetEnvPrefix("jira") + + // TODO: Change this to also work with EC2 Parameter Store (+KMS) + viper.BindEnv("url") + viper.BindEnv("username") + viper.BindEnv("password") + + u, err := url.Parse(viper.GetString("url")) + + if err != nil { + fmt.Fprintln(ctx.App.Writer, "Unable to parse JIRA URL: ", err) + return err + } + + viper.Set("url", u) + + return nil + +} + + +func replicate(ctx *cli.Context) error { + c := client.New(viper.Get("url").(*url.URL), viper.GetString("username"), viper.GetString("password")) + + for { + for { + err := makeBackup(ctx) + + if err != nil { + switch v := err.(type) { + case client.BackupRateExceeded: + fmt.Fprintln(ctx.App.Writer, "Backup rate exceeded. Retrying in", v.RetryIn()) + time.Sleep(v.RetryIn()) + default: + fmt.Fprintln(ctx.App.Writer, "Couldn't make a backup.") + return err + } + } + + break + } + + start := time.Now() + fmt.Fprintln(ctx.App.Writer, "Waiting for backup to be ready") + + waiter := time.NewTicker(30*time.Second) + + for { + <- waiter.C + + fmt.Fprintln(ctx.App.Writer,"Checking backup status") + status, err := c.GetBackupStatus() + + + if err != nil { + fmt.Fprintln(ctx.App.Writer, "Unable to get backup status: ", err) + } + + if status.Status != "InProgress" { + break + } + } + + + fmt.Fprintln(ctx.App.Writer,"Copying backup to S3") + err := backupToS3(ctx) + duration := time.Now().Sub(start) + + if err != nil { + fmt.Fprintln(ctx.App.Writer, "Unable to copy backup to S3") + } else { + fmt.Fprintln(ctx.App.Writer, "Backup and copy to S3 completed. Took", duration) + fmt.Fprintln(ctx.App.Writer, "Starting a new backup in 48 hours at:", time.Now().Add(48*time.Hour).Format(time.RFC822Z)) + time.Sleep(48*time.Hour) + } + } +} + +func downloadBackup(ctx *cli.Context) error { + output := ctx.String("o") + if output == "" { + fmt.Fprintln(ctx.App.Writer, "Output file path is required") + return errors.New("output file path is required") + } + + file, err := os.OpenFile(output, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + + if err != nil { + fmt.Fprintln(ctx.App.Writer, "Unable to open download file: ", err) + return err + } + + c := client.New(viper.Get("url").(*url.URL), viper.GetString("username"), viper.GetString("password")) + + reader, length, err := c.DownloadBackup() + + if err != nil { + fmt.Fprintln(ctx.App.Writer, "Unable to get backup: ", err) + return err + } + + // Create a Tee reader to do some + bar := pb.New(length).SetUnits(pb.U_BYTES) + bar.Start() + + br := bar.NewProxyReader(reader) + io.Copy(file, br) + bar.Finish() + + return err + +} + +func makeBackup(ctx *cli.Context) error { + c := client.New(viper.Get("url").(*url.URL), viper.GetString("username"), viper.GetString("password")) + err := c.CreateBackup(true) + + if err != nil { + switch v := err.(type) { + case client.BackupRateExceeded: + fmt.Fprintln(ctx.App.Writer, "Backup rate exceeded. Try again at:", v.RetryAt()) + default: + fmt.Fprintln(ctx.App.Writer, "Unable to create backup:", err) + } + + } + + return err +} + +func backupToS3(ctx *cli.Context) error { + + l := log.New(ctx.App.Writer, "s3-backup ", log.LstdFlags) + + viper.SetConfigFile(ctx.String("config")) + err := viper.ReadInConfig() + + if err != nil { + l.Println("Unable to read configuration file.") + return nil + } + + c := client.New(viper.Get("url").(*url.URL), viper.GetString("username"), viper.GetString("password")) + + reader, size, err := c.DownloadBackup() + + if err != nil { + l.Println("Unable to get backup: ", err) + return nil + } + + awsSession := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(viper.GetString("s3.region")), + })) + + s3Client := s3.New(awsSession) + + bucketName := viper.GetString("s3.bucket") + keyName := fmt.Sprintf("jira-%s.zip", time.Now().Format("2006-01-02")) + + uploadReq := &s3.CreateMultipartUploadInput{ + Bucket: &bucketName, + Key: &keyName, + ContentType: aws.String("application/zip"), + ACL: aws.String(s3.BucketCannedACLPrivate), + } + + if storageClass := viper.GetString("s3.storage_class"); storageClass != "" { + uploadReq.StorageClass = &storageClass + } + + if kmsKeyId := viper.GetString("kms.key"); kmsKeyId != "" { + uploadReq.ServerSideEncryption = aws.String(s3.ServerSideEncryptionAwsKms) + uploadReq.SSEKMSKeyId = &kmsKeyId + } + + upload, err := s3Client.CreateMultipartUpload(uploadReq) + + if err != nil { + l.Println("Unable to create multi-part upload:", err) + return err + } + + i := 0 + + var partMap []*s3.CompletedPart + + bar := pb.New(size).SetUnits(pb.U_BYTES) + + barReader := bar.NewProxyReader(reader) + + bar.Start() + + nParts := int(math.Ceil(float64(size) / float64(partSize))) + + + + for partNumber := int64(1); i < size; partNumber++ { + bar.Postfix( fmt.Sprintf(" %02d/%02d", partNumber, nParts) ) + // Read a part's worth of data from JIRA + store := make([]byte, partSize) // 16MiB + + bytesRead, err := io.ReadFull(barReader, store) + + if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { + l.Println("Unable to read part:", err) + return err + } + + if err == io.EOF && bytesRead == 0 { + break + } + + var bufferReader *bytes.Reader + if bytesRead == partSize { + + bufferReader = bytes.NewReader(store) + } else { + bufferReader = bytes.NewReader(store[:bytesRead]) + } + + + partReq := &s3.UploadPartInput{ + Bucket: &bucketName, + Key: &keyName, + UploadId: upload.UploadId, + PartNumber: &partNumber, + ContentLength: aws.Int64(int64(bytesRead)), + Body: bufferReader, + } + + partResult, err := s3Client.UploadPart(partReq) + + if err != nil { + l.Println("Unable to upload part:", err) + return err + } + + partMap = append(partMap, &s3.CompletedPart{ + PartNumber: aws.Int64(partNumber), + ETag: partResult.ETag, + }) + } + + bar.Finish() + + l.Println("Completing Upload.") + + completedUpload := &s3.CompletedMultipartUpload{ + Parts: partMap, + } + + completionReq := &s3.CompleteMultipartUploadInput{ + Bucket: &bucketName, + Key: &keyName, + UploadId: upload.UploadId, + MultipartUpload: completedUpload, + } + + completion, err := s3Client.CompleteMultipartUpload(completionReq) + + if err != nil { + l.Println("Unable to complete upload:", err) + } + + l.Println("Completed upload:", *completion.ETag) + + + return nil +} \ No newline at end of file