From e5bde2dd64945929f132388b28e5cb481bff50e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= Date: Sun, 27 Feb 2022 23:56:05 +0100 Subject: [PATCH 1/2] feat: Add health check endpoint (https://github.com/riotkit-org/backup-repository/issues/184) --- server-go/Makefile | 3 ++ server-go/core/ctx.go | 3 ++ server-go/docs/README.md | 3 +- server-go/docs/api/administrative/README.md | 37 ++++++++++++++++++++ server-go/go.mod | 6 ++-- server-go/health/backupwindow.go | 13 +++---- server-go/health/db.go | 22 ++++++++++++ server-go/health/interface.go | 7 ++-- server-go/health/size.go | 15 ++++---- server-go/health/storage.go | 23 +++++++++++++ server-go/health/sumofversions.go | 11 +++--- server-go/http/collection.go | 8 ++--- server-go/http/health.go | 38 +++++++++++++++++++++ server-go/http/main.go | 5 +++ server-go/main.go | 3 ++ server-go/storage/service.go | 24 +++++++++++++ 16 files changed, 190 insertions(+), 31 deletions(-) create mode 100644 server-go/docs/api/administrative/README.md create mode 100644 server-go/health/db.go create mode 100644 server-go/health/storage.go create mode 100644 server-go/http/health.go diff --git a/server-go/Makefile b/server-go/Makefile index c891a7f9..48babf48 100644 --- a/server-go/Makefile +++ b/server-go/Makefile @@ -1,5 +1,8 @@ all: build run +test_health: + curl -s -X GET 'http://localhost:8080/health' + test_login: curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' @echo "Now do export TOKEN=..." diff --git a/server-go/core/ctx.go b/server-go/core/ctx.go index d7c64a80..ece54b28 100644 --- a/server-go/core/ctx.go +++ b/server-go/core/ctx.go @@ -7,14 +7,17 @@ import ( "github.com/riotkit-org/backup-repository/security" "github.com/riotkit-org/backup-repository/storage" "github.com/riotkit-org/backup-repository/users" + "gorm.io/gorm" ) type ApplicationContainer struct { + Db *gorm.DB Config *config.ConfigurationProvider Users *users.Service GrantedAccesses *security.Service Collections *collections.Service Storage *storage.Service JwtSecretKey string + HealthCheckKey string Locks *concurrency.LocksService } diff --git a/server-go/docs/README.md b/server-go/docs/README.md index b30404b0..75d0b0e8 100644 --- a/server-go/docs/README.md +++ b/server-go/docs/README.md @@ -36,5 +36,4 @@ Interactions with server are done using HTTP API that talks JSON in both ways, a ### Collections -### Administrative - +### [Administrative](api/administrative/README.md) diff --git a/server-go/docs/api/administrative/README.md b/server-go/docs/api/administrative/README.md new file mode 100644 index 00000000..74fe8bb4 --- /dev/null +++ b/server-go/docs/api/administrative/README.md @@ -0,0 +1,37 @@ +Administrative API endpoints +============================ + +## GET `/health`` + +**Example:** + +```bash +curl -s -X GET 'http://localhost:8080/health' +``` + +**Example response (200):** + +```json +{ + "data": { + "health": [ + { + "message": "OK", + "name": "DbValidator", + "status": true, + "statusText": "DbValidator=true" + }, + { + "message": "OK", + "name": "StorageAvailabilityValidator", + "status": true, + "statusText": "StorageAvailabilityValidator=true" + } + ] + }, + "status": true +} +``` + +**Other responses:** +- [500](../common-responses.md) diff --git a/server-go/go.mod b/server-go/go.mod index 19a1f96c..5a2060ea 100644 --- a/server-go/go.mod +++ b/server-go/go.mod @@ -5,10 +5,13 @@ go 1.17 require ( github.com/appleboy/gin-jwt/v2 v2.8.0 github.com/fatih/structs v1.1.0 + github.com/gin-contrib/timeout v0.0.3 github.com/gin-gonic/gin v1.7.7 github.com/google/uuid v1.3.0 github.com/jessevdk/go-flags v1.5.0 github.com/julianshen/gin-limiter v0.0.0-20161123033831-fc39b5e90fe7 + github.com/labstack/gommon v0.3.1 + github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 @@ -38,7 +41,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-contrib/timeout v0.0.3 // indirect github.com/go-logr/logr v1.2.0 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect @@ -66,13 +68,11 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/ratelimit v1.0.1 // indirect - github.com/labstack/gommon v0.3.1 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/server-go/health/backupwindow.go b/server-go/health/backupwindow.go index e6a96768..5e732281 100644 --- a/server-go/health/backupwindow.go +++ b/server-go/health/backupwindow.go @@ -11,10 +11,11 @@ import ( type BackupWindowValidator struct { svc *storage.Service + c *collections.Collection } -func (v BackupWindowValidator) Validate(c *collections.Collection) error { - latest, err := v.svc.FindLatestVersion(c.GetId()) +func (v BackupWindowValidator) Validate() error { + latest, err := v.svc.FindLatestVersion(v.c.GetId()) if err != nil { return err } @@ -23,11 +24,11 @@ func (v BackupWindowValidator) Validate(c *collections.Collection) error { now := time.Now() // Backup Windows are optional - if len(c.Spec.Windows) == 0 { + if len(v.c.Spec.Windows) == 0 { return nil } - for _, window := range c.Spec.Windows { + for _, window := range v.c.Spec.Windows { matches, err := window.IsInPreviousWindowTimeSlot(now, latest.CreatedAt) if err != nil { return errors.New(fmt.Sprintf("failed to calculate previous run for window '%v' - %v", window, err)) @@ -47,6 +48,6 @@ func (v BackupWindowValidator) Validate(c *collections.Collection) error { return errors.Errorf("previous backup was not executed in expected time slots: %v", strings.Trim(allowedSlots, ", ")) } -func NewBackupWindowValidator(svc *storage.Service) BackupWindowValidator { - return BackupWindowValidator{svc} +func NewBackupWindowValidator(svc *storage.Service, c *collections.Collection) BackupWindowValidator { + return BackupWindowValidator{svc, c} } diff --git a/server-go/health/db.go b/server-go/health/db.go new file mode 100644 index 00000000..32a0be9a --- /dev/null +++ b/server-go/health/db.go @@ -0,0 +1,22 @@ +package health + +import ( + "github.com/pkg/errors" + "gorm.io/gorm" +) + +type DbValidator struct { + db *gorm.DB +} + +func (v DbValidator) Validate() error { + err := v.db.Raw("SELECT 1").Error + if err != nil { + return errors.Wrapf(err, "cannot connect to database") + } + return nil +} + +func NewDbValidator(db *gorm.DB) DbValidator { + return DbValidator{db} +} diff --git a/server-go/health/interface.go b/server-go/health/interface.go index 5942065f..a4d58bb0 100644 --- a/server-go/health/interface.go +++ b/server-go/health/interface.go @@ -3,20 +3,19 @@ package health import ( "encoding/json" "fmt" - "github.com/riotkit-org/backup-repository/collections" "reflect" ) type Validator interface { - Validate(c *collections.Collection) error + Validate() error } type Validators []Validator -func (v Validators) Validate(c *collections.Collection) StatusCollection { +func (v Validators) Validate() StatusCollection { var status StatusCollection for _, validator := range v { - if err := validator.Validate(c); err != nil { + if err := validator.Validate(); err != nil { status = append(status, Status{ Name: reflect.TypeOf(validator).Name(), StatusMsg: err.Error(), diff --git a/server-go/health/size.go b/server-go/health/size.go index 1f4500f7..97c89a9d 100644 --- a/server-go/health/size.go +++ b/server-go/health/size.go @@ -8,17 +8,18 @@ import ( type VersionsSizeValidator struct { svc *storage.Service + c *collections.Collection } -func (v VersionsSizeValidator) Validate(c *collections.Collection) error { - versions, err := v.svc.FindAllActiveVersionsFor(c.GetId()) +func (v VersionsSizeValidator) Validate() error { + versions, err := v.svc.FindAllActiveVersionsFor(v.c.GetId()) if err != nil { - return errors.Wrapf(err, "Cannot list versions for collection id=%v", c.GetId()) + return errors.Wrapf(err, "Cannot list versions for collection id=%v", v.c.GetId()) } - maxVersionSize, err := c.GetMaxOneVersionSizeInBytes() + maxVersionSize, err := v.c.GetMaxOneVersionSizeInBytes() if err != nil { - return errors.Wrapf(err, "Cannot list versions for collection id=%v", c.GetId()) + return errors.Wrapf(err, "Cannot list versions for collection id=%v", v.c.GetId()) } for _, v := range versions { @@ -30,6 +31,6 @@ func (v VersionsSizeValidator) Validate(c *collections.Collection) error { return nil } -func NewVersionsSizeValidator(svc *storage.Service) VersionsSizeValidator { - return VersionsSizeValidator{svc} +func NewVersionsSizeValidator(svc *storage.Service, c *collections.Collection) VersionsSizeValidator { + return VersionsSizeValidator{svc, c} } diff --git a/server-go/health/storage.go b/server-go/health/storage.go new file mode 100644 index 00000000..dad3eae7 --- /dev/null +++ b/server-go/health/storage.go @@ -0,0 +1,23 @@ +package health + +import ( + "github.com/pkg/errors" + "github.com/riotkit-org/backup-repository/storage" +) + +type StorageAvailabilityValidator struct { + storage *storage.Service +} + +func (v StorageAvailabilityValidator) Validate() error { + err := v.storage.TestReadWrite() + if err != nil { + return errors.Wrapf(err, "storage not operable") + } + + return nil +} + +func NewStorageValidator(storage *storage.Service) StorageAvailabilityValidator { + return StorageAvailabilityValidator{storage} +} diff --git a/server-go/health/sumofversions.go b/server-go/health/sumofversions.go index bf649590..7c2137e4 100644 --- a/server-go/health/sumofversions.go +++ b/server-go/health/sumofversions.go @@ -8,17 +8,18 @@ import ( type SumOfVersionsValidator struct { svc *storage.Service + c *collections.Collection } -func (v SumOfVersionsValidator) Validate(c *collections.Collection) error { +func (v SumOfVersionsValidator) Validate() error { var totalSize int64 - allActive, _ := v.svc.FindAllActiveVersionsFor(c.GetId()) + allActive, _ := v.svc.FindAllActiveVersionsFor(v.c.GetId()) for _, version := range allActive { totalSize += version.Filesize } - maxCollectionSize, _ := c.GetCollectionMaxSize() + maxCollectionSize, _ := v.c.GetCollectionMaxSize() if totalSize > maxCollectionSize { return errors.Errorf("Summary of all files is %vb, while collection hard limit is %vb", totalSize, maxCollectionSize) @@ -27,6 +28,6 @@ func (v SumOfVersionsValidator) Validate(c *collections.Collection) error { return nil } -func NewSumOfVersionsValidator(svc *storage.Service) SumOfVersionsValidator { - return SumOfVersionsValidator{svc} +func NewSumOfVersionsValidator(svc *storage.Service, c *collections.Collection) SumOfVersionsValidator { + return SumOfVersionsValidator{svc, c} } diff --git a/server-go/http/collection.go b/server-go/http/collection.go index 9721c122..5664ea7b 100644 --- a/server-go/http/collection.go +++ b/server-go/http/collection.go @@ -202,10 +202,10 @@ func addCollectionHealthRoute(r *gin.Engine, ctx *core.ApplicationContainer, rat // Run all the checks healthStatuses := health.Validators{ - health.NewBackupWindowValidator(ctx.Storage), - health.NewVersionsSizeValidator(ctx.Storage), - health.NewSumOfVersionsValidator(ctx.Storage), - }.Validate(collection) + health.NewBackupWindowValidator(ctx.Storage, collection), + health.NewVersionsSizeValidator(ctx.Storage, collection), + health.NewSumOfVersionsValidator(ctx.Storage, collection), + }.Validate() if !healthStatuses.GetOverallStatus() { ServerErrorResponseWithData(c, errors.New("one of checks failed"), gin.H{ diff --git a/server-go/http/health.go b/server-go/http/health.go new file mode 100644 index 00000000..9d3b33b5 --- /dev/null +++ b/server-go/http/health.go @@ -0,0 +1,38 @@ +package http + +import ( + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + "github.com/riotkit-org/backup-repository/core" + "github.com/riotkit-org/backup-repository/health" +) + +func addServerHealthEndpoint(r *gin.Engine, ctx *core.ApplicationContainer, rateLimiter gin.HandlerFunc) { + r.GET("/health", rateLimiter, func(c *gin.Context) { + // Authorization + healthCode := c.GetHeader("Authorization") + if healthCode == "" { + healthCode = c.Query("code") + } + if healthCode != ctx.HealthCheckKey { + UnauthorizedResponse(c, errors.New("health code invalid. Should be provided withing 'Authorization' header or 'code' query string. Must match --health-check-code commandline switch value")) + return + } + + healthStatuses := health.Validators{ + health.NewDbValidator(ctx.Db), + health.NewStorageValidator(ctx.Storage), + }.Validate() + + if !healthStatuses.GetOverallStatus() { + ServerErrorResponseWithData(c, errors.New("one of checks failed"), gin.H{ + "health": healthStatuses, + }) + return + } + + OKResponse(c, gin.H{ + "health": healthStatuses, + }) + }) +} diff --git a/server-go/http/main.go b/server-go/http/main.go index 2d7bbdb8..59a659e7 100644 --- a/server-go/http/main.go +++ b/server-go/http/main.go @@ -44,5 +44,10 @@ func SpawnHttpApplication(ctx *core.ApplicationContainer) { return "collectionHealth:" + ctx.ClientIP(), nil }).Middleware()) + // server health + addServerHealthEndpoint(r, ctx, limiter.NewRateLimiter(time.Minute, 10, func(ctx *gin.Context) (string, error) { + return "health:" + ctx.ClientIP(), nil + }).Middleware()) + _ = r.Run() } diff --git a/server-go/main.go b/server-go/main.go index 90567f87..288dfdd8 100644 --- a/server-go/main.go +++ b/server-go/main.go @@ -26,6 +26,7 @@ type options struct { DbName string `long:"db-name" description:"Database name inside a database"` DbPort int `long:"db-port" description:"Database name inside a database" default:"5432"` JwtSecretKey string `long:"jwt-secret-key" short:"s" description:"Secret used for generating JSON Web Tokens for authentication"` + HealthCheckKey string `long:"health-check-key" short:"k" description:"Secret key to access health check endpoint"` Level string `long:"log-level" description:"Log level" default:"debug"` StorageDriverUrl string `long:"storage-url" description:"Storage driver url compatible with GO Cloud (https://gocloud.dev/howto/blob/)"` IsGCS bool `long:"use-google-cloud" description:"If using Google Cloud Storage, then in --storage-url just type bucket name"` @@ -80,10 +81,12 @@ func main() { } ctx := core.ApplicationContainer{ + Db: dbDriver, Config: &configProvider, Users: &usersService, GrantedAccesses: &gaService, JwtSecretKey: opts.JwtSecretKey, + HealthCheckKey: opts.HealthCheckKey, Collections: &collectionsService, Storage: &storageService, Locks: &locksService, diff --git a/server-go/storage/service.go b/server-go/storage/service.go index b47b8dbd..030ea58c 100644 --- a/server-go/storage/service.go +++ b/server-go/storage/service.go @@ -212,6 +212,30 @@ func (s *Service) FindAllActiveVersionsFor(id string) ([]UploadedVersion, error) return s.repository.findAllActiveVersions(id) } +// TestReadWrite is performing a simple write & read & delete operation to check if storage is healthy +func (s *Service) TestReadWrite() error { + healthKey := fmt.Sprintf(".health-%v", time.Now().UnixNano()) + healthSecret := fmt.Sprintf("secret-%v", time.Now().Unix()) + + if err := s.storage.WriteAll(context.TODO(), healthKey, []byte(healthSecret), &blob.WriterOptions{}); err != nil { + return err + } + + read, err := s.storage.ReadAll(context.TODO(), healthKey) + if err != nil { + return err + } + if string(read) != healthSecret { + return errors.New("cannot verify storage read&write - wrote a text, but read a different text") + } + + if err := s.storage.Delete(context.TODO(), healthKey); err != nil { + return err + } + + return nil +} + // NewService is a factory method that knows how to construct a Storage provider, distincting multiple types of providers func NewService(db *gorm.DB, driverUrl string, isUsingGCS bool) (Service, error) { repository := VersionsRepository{db: db} From 872de632fcf5546b51ef365a00801d5d60e68c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= Date: Mon, 28 Feb 2022 22:46:44 +0100 Subject: [PATCH 2/2] feat: Add ConfigurationProvider health check (https://github.com/riotkit-org/backup-repository/issues/184) --- server-go/config/interface.go | 1 + server-go/config/kubernetes.go | 16 ++++++++++++++++ server-go/health/config.go | 22 ++++++++++++++++++++++ server-go/http/health.go | 1 + 4 files changed, 40 insertions(+) create mode 100644 server-go/health/config.go diff --git a/server-go/config/interface.go b/server-go/config/interface.go index dfa6c0c9..fd8abba7 100644 --- a/server-go/config/interface.go +++ b/server-go/config/interface.go @@ -5,4 +5,5 @@ type ConfigurationProvider interface { GetSingleDocumentAnyType(kind string, id string, apiGroup string, apiVersion string) (string, error) StoreDocument(kind string, document interface{}) error + GetHealth() error } diff --git a/server-go/config/kubernetes.go b/server-go/config/kubernetes.go index 0c74154a..17bc2036 100644 --- a/server-go/config/kubernetes.go +++ b/server-go/config/kubernetes.go @@ -3,6 +3,7 @@ package config import ( "context" "github.com/fatih/structs" + "github.com/pkg/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -18,6 +19,21 @@ type ConfigurationInKubernetes struct { apiVersion string } +func (o *ConfigurationInKubernetes) GetHealth() error { + resources := []schema.GroupVersionResource{ + {Group: o.apiGroup, Version: o.apiVersion, Resource: "backupusers"}, + {Group: o.apiGroup, Version: o.apiVersion, Resource: "backupcollections"}, + } + + for _, resource := range resources { + if _, err := o.api.Resource(resource).Namespace(o.namespace).List(context.Background(), metav1.ListOptions{}); err != nil { + return errors.Wrapf(err, "cannot access Kubrenetes resources: '%v'", resource.String()) + } + } + + return nil +} + func (o *ConfigurationInKubernetes) GetSingleDocumentAnyType(kind string, id string, apiGroup string, apiVersion string) (string, error) { resource := schema.GroupVersionResource{Group: apiGroup, Version: apiVersion, Resource: kind} object, err := o.api.Resource(resource).Namespace(o.namespace).Get(context.Background(), id, metav1.GetOptions{}) diff --git a/server-go/health/config.go b/server-go/health/config.go new file mode 100644 index 00000000..57560c81 --- /dev/null +++ b/server-go/health/config.go @@ -0,0 +1,22 @@ +package health + +import ( + "github.com/pkg/errors" + "github.com/riotkit-org/backup-repository/config" +) + +type ConfigurationProviderValidator struct { + cfg config.ConfigurationProvider +} + +func (v ConfigurationProviderValidator) Validate() error { + if err := v.cfg.GetHealth(); err != nil { + return errors.Wrapf(err, "configuration provider is not usable") + } + + return nil +} + +func NewConfigurationProviderValidator(cfg config.ConfigurationProvider) ConfigurationProviderValidator { + return ConfigurationProviderValidator{cfg} +} diff --git a/server-go/http/health.go b/server-go/http/health.go index 9d3b33b5..6d809e7b 100644 --- a/server-go/http/health.go +++ b/server-go/http/health.go @@ -22,6 +22,7 @@ func addServerHealthEndpoint(r *gin.Engine, ctx *core.ApplicationContainer, rate healthStatuses := health.Validators{ health.NewDbValidator(ctx.Db), health.NewStorageValidator(ctx.Storage), + health.NewConfigurationProviderValidator(*ctx.Config), }.Validate() if !healthStatuses.GetOverallStatus() {