ratelimiting (#48)

* ratelimiting implementation

.gitignore

@@ -1,5 +1,6 @@
 vendor/
-envoy-control-plane-darwin-amd64
+/*-darwin-amd64
+/*-linux-amd64
 resources/*/pki/
 resources/*/challenges/
 /*.sh

Makefile

@@ -1,10 +1,12 @@
 GOARCH = amd64
 SERVER_BINARY = envoy-control-plane
 
-build-darwin:
+build-darwin: build-server-darwin
+
+build-server-darwin:
 	GOOS=darwin GOARCH=${GOARCH} go build ${LDFLAGS} -o ${SERVER_BINARY}-darwin-${GOARCH} cmd/envoy-control-plane/main.go 
 
-build-linux:
+build-server-linux:
 	GOOS=linux GOARCH=${GOARCH} go build ${LDFLAGS} -o ${SERVER_BINARY}-linux-${GOARCH} cmd/envoy-control-plane/main.go 
 
 test:

README.md

@@ -7,6 +7,9 @@ Envoy autocert is an envoy control plane with AWS Cloud support.
 * ACME support to automatically verify, issue and setup letsencrypt certificates
 * authn: support for JWT authentication
 * authz: support for external grpc service which can authorize a connection
+* Access Log Server support
+* Compression support
+* Ratelimit support
 * Works stand-alone or serverless with AWS Fargate
 * Traffic only passes the envoy proxy
 

cmd/envoy-control-plane/main.go

@@ -8,28 +8,28 @@ import (
 	envoy "github.com/in4it/roxprox/pkg/envoy"
 	"github.com/in4it/roxprox/pkg/management"
 	storage "github.com/in4it/roxprox/pkg/storage"
-	localStorage "github.com/in4it/roxprox/pkg/storage/local"
-	"github.com/in4it/roxprox/pkg/storage/s3"
 	"github.com/juju/loggo"
 )
 
 var logger = loggo.GetLogger("envoy-control-plane")
 
 func main() {
 	var (
-		err           error
-		loglevel      string
-		storageType   string
-		storagePath   string
-		storageBucket string
-		awsRegion     string
-		acmeContact   string
-		s             storage.Storage
+		err                  error
+		loglevel             string
+		storageType          string
+		storagePath          string
+		storageBucket        string
+		storageNotifications string
+		awsRegion            string
+		acmeContact          string
+		s                    storage.Storage
 	)
 	flag.StringVar(&loglevel, "loglevel", "INFO", "log level")
 	flag.StringVar(&storageType, "storage-type", "local", "storage type")
 	flag.StringVar(&storagePath, "storage-path", "", "storage path")
 	flag.StringVar(&storageBucket, "storage-bucket", "", "s3 storage bucket")
+	flag.StringVar(&storageNotifications, "storage-notifications", "", "s3 storage notifications")
 	flag.StringVar(&awsRegion, "aws-region", "", "AWS region")
 	flag.StringVar(&acmeContact, "acme-contact", "", "acme contact for TLS certs")
 
@@ -44,20 +44,14 @@ func main() {
 	}
 
 	if storageType == "local" {
-		s, err = storage.NewStorage(storageType, localStorage.Config{Path: storagePath})
+		s, err = storage.NewLocalStorage(storagePath)
 		if err != nil {
 			logger.Errorf("Couldn't inialize storage: %s", err)
 			os.Exit(1)
 		}
 	} else if storageType == "s3" {
-		if storageBucket == "" {
-			logger.Errorf("No bucket specified")
-			os.Exit(1)
-		}
-		if strings.HasSuffix(storagePath, "/") {
-			storagePath = storagePath[:len(storagePath)-1]
-		}
-		s, err = storage.NewStorage(storageType, s3.Config{Prefix: storagePath, Bucket: storageBucket, Region: awsRegion})
+		startNotificationQueue := true
+		s, err = storage.NewS3Storage(storageBucket, storagePath, awsRegion, storageNotifications, startNotificationQueue)
 		if err != nil {
 			logger.Errorf("Couldn't inialize storage: %s", err)
 			os.Exit(1)

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/aws/aws-sdk-go v1.32.10
 	github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 // indirect
+	github.com/coocood/freecache v1.1.1
 	github.com/envoyproxy/go-control-plane v0.9.6
 	github.com/ghodss/yaml v1.0.0
 	github.com/gogo/protobuf v1.3.1

go.sum

@@ -1,18 +1,23 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/aws/aws-sdk-go v1.20.5 h1:Ytq5AxpA2pr4vRJM9onvgAjjVRZKKO63WStbG/jLHw0=
 github.com/aws/aws-sdk-go v1.20.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.32.10 h1:cEJTxGcBGlsM2tN36MZQKhlK93O9HrnaRs+lq2f0zN8=
 github.com/aws/aws-sdk-go v1.32.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533 h1:8wZizuKuZVu5COB7EsBYxBQz8nRcXXn5d4Gt91eJLvU=
 github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc=
+github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/envoy v1.14.1 h1:iqLWWa0bsImtur3PIm59jEczqu8/7q8JbNx1eFidLJA=
@@ -92,6 +97,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=

pkg/api/ratelimit.go

@@ -0,0 +1,19 @@
+package api
+
+type RateLimit struct {
+	API      string        `json:"api" yaml:"api"`
+	Kind     string        `json:"kind" yaml:"kind"`
+	Metadata Metadata      `json:"metadata" yaml:"metadata"`
+	Spec     RateLimitSpec `json:"spec" yaml:"spec"`
+}
+type RateLimitSpec struct {
+	Descriptors    []RateLimitDescriptor `json:"descriptors" yaml:"descriptors"`
+	RequestPerUnit string                `json:"requestPerUnit" yaml:"requestPerUnit"`
+	Unit           string                `json:"unit" yaml:"unit"`
+}
+type RateLimitDescriptor struct {
+	DestinationCluster bool   `json:"destinationCluster" yaml:"destinationCluster"`
+	SourceCluster      bool   `json:"sourceCluster" yaml:"sourceCluster"`
+	RemoteAddress      bool   `json:"remoteAddress" yaml:"remoteAddress"`
+	RequestHeader      string `json:"requestHeader" yaml:"requestHeader"`
+}

pkg/envoy/accesslogserver.go

@@ -35,7 +35,7 @@ func (c *AccessLogServer) updateListenersWithAccessLogServer(cache *WorkQueueCac
 				manager.AccessLog = accessLogConfig
 
 				// update manager in cache
-				pbst, err := ptypes.MarshalAny(&manager)
+				pbst, err := ptypes.MarshalAny(manager)
 				if err != nil {
 					return err
 				}

pkg/envoy/authzfilter.go

@@ -38,7 +38,7 @@ func (a *AuthzFilter) updateListenersWithAuthzFilter(cache *WorkQueueCache, para
 				updateHTTPFilterWithConfig(&manager.HttpFilters, "envoy.ext_authz", authzConfigEncoded)
 
 				// update manager in cache
-				pbst, err := ptypes.MarshalAny(&manager)
+				pbst, err := ptypes.MarshalAny(manager)
 				if err != nil {
 					return err
 				}

pkg/envoy/compression.go

@@ -28,7 +28,7 @@ func (c *Compression) updateListenersWithCompression(cache *WorkQueueCache, para
 					return err
 				}
 
-				// get authz config config
+				// get compression config
 				compressorConfigEncoded, err := c.getCompressionFilterEncoded(params)
 				if err != nil {
 					return err
@@ -38,7 +38,7 @@ func (c *Compression) updateListenersWithCompression(cache *WorkQueueCache, para
 				updateHTTPFilterWithConfig(&manager.HttpFilters, "envoy.filters.http.compressor", compressorConfigEncoded)
 
 				// update manager in cache
-				pbst, err := ptypes.MarshalAny(&manager)
+				pbst, err := ptypes.MarshalAny(manager)
 				if err != nil {
 					return err
 				}

pkg/envoy/jwtprovider.go

@@ -180,7 +180,7 @@ func (j *JwtProvider) updateListenerWithJwtProvider(cache *WorkQueueCache, param
 			return err
 		}
 		// add routes to jwtProvider
-		var jwtConfig jwtAuth.JwtAuthentication
+		jwtConfig := &jwtAuth.JwtAuthentication{}
 		if getListenerHTTPFilterIndex("envoy.filters.http.jwt_authn", manager.HttpFilters) != -1 {
 			jwtConfig, err = getListenerHTTPFilterJwtAuth(manager.HttpFilters)
 			if err != nil {
@@ -194,14 +194,14 @@ func (j *JwtProvider) updateListenerWithJwtProvider(cache *WorkQueueCache, param
 		jwtConfig.Providers[params.Auth.JwtProvider] = jwtNewConfig.Providers[params.Auth.JwtProvider]
 		logger.Debugf("Adding/updating %s to jwt config", params.Auth.JwtProvider)
 
-		jwtConfigEncoded, err := ptypes.MarshalAny(&jwtConfig)
+		jwtConfigEncoded, err := ptypes.MarshalAny(jwtConfig)
 		if err != nil {
 			panic(err)
 		}
 
 		updateHTTPFilterWithConfig(&manager.HttpFilters, "envoy.filters.http.jwt_authn", jwtConfigEncoded)
 
-		pbst, err := ptypes.MarshalAny(&manager)
+		pbst, err := ptypes.MarshalAny(manager)
 		if err != nil {
 			panic(err)
 		}
@@ -231,7 +231,7 @@ func (j *JwtProvider) UpdateJwtRule(cache *WorkQueueCache, params ListenerParams
 	}
 
 	// update listener
-	var manager hcm.HttpConnectionManager
+	var manager *hcm.HttpConnectionManager
 	var err error
 
 	ll := cache.listeners[listenerKey].(*api.Listener)
@@ -246,7 +246,7 @@ func (j *JwtProvider) UpdateJwtRule(cache *WorkQueueCache, params ListenerParams
 	}
 
 	// add routes to jwtProvider
-	var jwtConfig jwtAuth.JwtAuthentication
+	jwtConfig := &jwtAuth.JwtAuthentication{}
 	if getListenerHTTPFilterIndex("envoy.filters.http.jwt_authn", manager.HttpFilters) != -1 {
 		jwtConfig, err = getListenerHTTPFilterJwtAuth(manager.HttpFilters)
 		if err != nil {
@@ -282,14 +282,14 @@ func (j *JwtProvider) UpdateJwtRule(cache *WorkQueueCache, params ListenerParams
 			jwtConfig.Rules = append(jwtConfig.Rules, newJwtRule)
 		}
 	}
-	jwtConfigEncoded, err := ptypes.MarshalAny(&jwtConfig)
+	jwtConfigEncoded, err := ptypes.MarshalAny(jwtConfig)
 	if err != nil {
 		panic(err)
 	}
 
 	updateHTTPFilterWithConfig(&manager.HttpFilters, "envoy.filters.http.jwt_authn", jwtConfigEncoded)
 
-	pbst, err := ptypes.MarshalAny(&manager)
+	pbst, err := ptypes.MarshalAny(manager)
 	if err != nil {
 		panic(err)
 	}
@@ -332,7 +332,7 @@ func (j *JwtProvider) DeleteJwtRule(cache *WorkQueueCache, params ListenerParams
 	}
 
 	// http listener
-	var manager hcm.HttpConnectionManager
+	var manager *hcm.HttpConnectionManager
 	var err error
 
 	var ll *api.Listener
@@ -359,7 +359,7 @@ func (j *JwtProvider) DeleteJwtRule(cache *WorkQueueCache, params ListenerParams
 			index := j.requirementRuleIndex(jwtConfig.Rules, rule)
 			jwtConfig.Rules = append(jwtConfig.Rules[:index], jwtConfig.Rules[index+1:]...)
 		}
-		jwtConfigEncoded, err := ptypes.MarshalAny(&jwtConfig)
+		jwtConfigEncoded, err := ptypes.MarshalAny(jwtConfig)
 		if err != nil {
 			panic(err)
 		}
@@ -369,7 +369,7 @@ func (j *JwtProvider) DeleteJwtRule(cache *WorkQueueCache, params ListenerParams
 		logger.Debugf("Couldn't find jwt provider %s during deleteRoute", params.Auth.JwtProvider)
 	}
 
-	pbst, err := ptypes.MarshalAny(&manager)
+	pbst, err := ptypes.MarshalAny(manager)
 	if err != nil {
 		panic(err)
 	}