From 6b9d14a400108c999e185e57c43fe3d2d8cfa14c Mon Sep 17 00:00:00 2001 From: Robert Lippens Date: Thu, 30 Apr 2020 22:36:33 -0700 Subject: [PATCH] bug fixes, actually try out gcp/azure flows --- README.md | 39 ++++++++++++++++++++++ config/config.go | 2 +- snapshot_agent/agent.go | 8 ++++- snapshot_agent/azure.go | 4 ++- snapshot_agent/gcp.go | 72 +++++++++++++++++++++++------------------ snapshot_agent/s3.go | 3 ++ 6 files changed, 93 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 3339bc8..e1582b0 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,45 @@ In this way, the daemon will always run on the leader Raft node. Another way to do this, which would allow us to run the snapshot agent anywhere, is to simply have the daemons form their own Raft cluster, but this approach seemed much more cumbersome. +## Running + +The recommended way of running this daemon is using systemctl, since it handles restarts and failure scenarios quite well. To learn more about systemctl, checkout [this article](https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units). begin, create the following file at `/etc/systemd/system/snapshot.service`: + +``` +[Unit] +Description="An Open Source Snapshot Service for Raft" +Documentation=https://github.com/Lucretius/vault_raft_snapshot_agent/ +Requires=network-online.target +After=network-online.target +ConditionFileNotEmpty=/etc/vault.d/snapshot.json + +[Service] +Type=simple +User=vault +Group=vault +ExecStart=/usr/local/bin/vault_raft_snapshot_agent +ExecReload=/usr/local/bin/vault_raft_snapshot_agent +KillMode=process +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +Your configuration is assumed to exist at `/etc/vault.d/snapshot.json` and the actual daemon binary at `/usr/local/bin/vault_raft_snapshot_agent`. + +Then just run: + +``` +sudo systemctl enable snapshot +sudo systemctl start snapshot +``` + +If your configuration is right and Vault is running on the same host as the agent you will see one of the following: + +`Not running on leader node, skipping.` or `Successfully created snapshot to `, depending on if the daemon runs on the leader's host or not. + ## Configuration `addr` The address of the Vault cluster. This is used to check the Vault cluster leader IP, as well as generate snapshots. diff --git a/config/config.go b/config/config.go index 5275ded..d5073a1 100644 --- a/config/config.go +++ b/config/config.go @@ -17,7 +17,7 @@ type Configuration struct { AWS S3Config `json:"aws_storage"` Local LocalConfig `json:"local_storage"` GCP GCPConfig `json:"google_storage"` - Azure AzureConfig `json:"azure_blob_storage"` + Azure AzureConfig `json:"azure_storage"` RoleID string `json:"role_id"` SecretID string `json:"secret_id"` } diff --git a/snapshot_agent/agent.go b/snapshot_agent/agent.go index e28cf25..09d0c1a 100644 --- a/snapshot_agent/agent.go +++ b/snapshot_agent/agent.go @@ -42,7 +42,13 @@ func NewSnapshotter(config *config.Configuration) (*Snapshotter, error) { } } if config.GCP.Bucket != "" { - err = snapshotter.ConfigureS3(config) + err = snapshotter.ConfigureGCP(config) + if err != nil { + return nil, err + } + } + if config.Azure.ContainerName != "" { + err = snapshotter.ConfigureAzure(config) if err != nil { return nil, err } diff --git a/snapshot_agent/azure.go b/snapshot_agent/azure.go index 3252250..038d362 100644 --- a/snapshot_agent/azure.go +++ b/snapshot_agent/azure.go @@ -38,7 +38,9 @@ func (s *Snapshotter) CreateAzureSnapshot(reader io.ReadWriter, config *config.C return o1.Properties.LastModified.Before(o2.Properties.LastModified) } AzureBy(timestamp).Sort(blobs) - + if len(blobs)-int(config.Retain) <= 0 { + return url, nil + } blobsToDelete := blobs[0 : len(blobs)-int(config.Retain)] for _, b := range blobsToDelete { diff --git a/snapshot_agent/gcp.go b/snapshot_agent/gcp.go index 34aa4bc..815f73f 100644 --- a/snapshot_agent/gcp.go +++ b/snapshot_agent/gcp.go @@ -1,9 +1,9 @@ package snapshot_agent import ( + "bytes" "context" "fmt" - "io" "log" "sort" @@ -13,48 +13,56 @@ import ( ) // CreateGCPSnapshot writes snapshot to google storage -func (s *Snapshotter) CreateGCPSnapshot(reader io.ReadWriter, config *config.Configuration, currentTs int64) (string, error) { +func (s *Snapshotter) CreateGCPSnapshot(b *bytes.Buffer, config *config.Configuration, currentTs int64) (string, error) { fileName := fmt.Sprintf("raft_snapshot-%d.snap", currentTs) obj := s.GCPBucket.Object(fileName) w := obj.NewWriter(context.Background()) - if _, err := io.Copy(w, reader); err != nil { + + if _, err := w.Write(b.Bytes()); err != nil { return "", err - } else { - if config.Retain > 0 { - deleteCtx := context.Background() - query := &storage.Query{Prefix: "raft_snapshot-"} - it := s.GCPBucket.Objects(deleteCtx, query) - var files []storage.ObjectAttrs - for { - attrs, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - log.Println("Unable to iterate through bucket to find old snapshots to delete") - return fileName, err - } - files = append(files, *attrs) - } + } + + if err := w.Close(); err != nil { + return "", err + } - timestamp := func(o1, o2 *storage.ObjectAttrs) bool { - return o1.Updated.Before(o2.Updated) + if config.Retain > 0 { + deleteCtx := context.Background() + query := &storage.Query{Prefix: "raft_snapshot-"} + it := s.GCPBucket.Objects(deleteCtx, query) + var files []storage.ObjectAttrs + for { + attrs, err := it.Next() + if err == iterator.Done { + break } + if err != nil { + log.Println("Unable to iterate through bucket to find old snapshots to delete") + return fileName, err + } + files = append(files, *attrs) + } - GCPBy(timestamp).Sort(files) - snapshotsToDelete := files[0 : len(files)-int(config.Retain)] + timestamp := func(o1, o2 *storage.ObjectAttrs) bool { + return o1.Updated.Before(o2.Updated) + } + + GCPBy(timestamp).Sort(files) + if len(files)-int(config.Retain) <= 0 { + return fileName, nil + } + snapshotsToDelete := files[0 : len(files)-int(config.Retain)] - for _, ss := range snapshotsToDelete { - obj := s.GCPBucket.Object(ss.Name) - err = obj.Delete(deleteCtx) - if err != nil { - log.Println("Cannot delete old snapshot") - return fileName, err - } + for _, ss := range snapshotsToDelete { + obj := s.GCPBucket.Object(ss.Name) + err := obj.Delete(deleteCtx) + if err != nil { + log.Println("Cannot delete old snapshot") + return fileName, err } } - return fileName, nil } + return fileName, nil } // implementation of Sort interface for s3 objects diff --git a/snapshot_agent/s3.go b/snapshot_agent/s3.go index 944440f..87f64e0 100644 --- a/snapshot_agent/s3.go +++ b/snapshot_agent/s3.go @@ -64,6 +64,9 @@ func (s *Snapshotter) CreateS3Snapshot(reader io.ReadWriter, config *config.Conf return o1.LastModified.Before(*o2.LastModified) } S3By(timestamp).Sort(existingSnapshots) + if len(existingSnapshots)-int(config.Retain) <= 0 { + return o.Location, nil + } snapshotsToDelete := existingSnapshots[0 : len(existingSnapshots)-int(config.Retain)] for i := range snapshotsToDelete {