Allow multiple webhook secrets to allow for zero-downtime rotation

azure/azevents/azevents.go

@@ -23,10 +23,10 @@ type Config struct {
 	Subscription  string
 	ResourceGroup string
 
-	// AuthSecret is a shared random secret between the event subscription and this
-	// receiver, to prevent random unauthenticated internet requests from triggering
-	// webhooks.
-	AuthSecret string
+	// AllowedAuthSecrets is a list of shared random secrets that will be accepted
+	// for incoming webhooks from the event subscription to this receiver, to
+	// prevent random unauthenticated internet requests from triggering webhooks.
+	AllowedAuthSecrets []string
 
 	ProcessedPortfolioTopicName string
 }
@@ -41,8 +41,8 @@ func (c *Config) validate() error {
 	if c.ResourceGroup == "" {
 		return errors.New("no resource group given")
 	}
-	if c.AuthSecret == "" {
-		return errors.New("no auth secret was given")
+	if len(c.AllowedAuthSecrets) == 0 {
+		return errors.New("no auth secrets were given")
 	}
 	if c.ProcessedPortfolioTopicName == "" {
 		return errors.New("no resource group given")
@@ -54,7 +54,7 @@ func (c *Config) validate() error {
 type Server struct {
 	logger *zap.Logger
 
-	authSecret string
+	allowedAuthSecrets []string
 
 	subscription  string
 	resourceGroup string
@@ -67,16 +67,25 @@ func NewServer(cfg *Config) (*Server, error) {
 	}
 
 	return &Server{
-		logger:        cfg.Logger,
-		authSecret:    cfg.AuthSecret,
-		subscription:  cfg.Subscription,
-		resourceGroup: cfg.ResourceGroup,
+		logger:             cfg.Logger,
+		allowedAuthSecrets: cfg.AllowedAuthSecrets,
+		subscription:       cfg.Subscription,
+		resourceGroup:      cfg.ResourceGroup,
 		pathToTopic: map[string]string{
 			"/events/processed_portfolio": cfg.ProcessedPortfolioTopicName,
 		},
 	}, nil
 }
 
+func (s *Server) findValidAuthSecret(auth string) (int, bool) {
+	for idx, allowed := range s.allowedAuthSecrets {
+		if auth == allowed {
+			return idx, true
+		}
+	}
+	return 0, false
+}
+
 func (s *Server) verifyWebhook(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		topic, ok := s.pathToTopic[r.URL.Path]
@@ -87,11 +96,22 @@ func (s *Server) verifyWebhook(next http.Handler) http.Handler {
 		}
 
 		if r.Header.Get("aeg-event-type") != "SubscriptionValidation" {
-			if auth := r.Header.Get("authorization"); auth != s.authSecret {
-				s.logger.Error("missing or invalid auth", zap.String("invalid_auth", auth))
+			// Azure doesn't have a native way to validate where a webhook request
+			// came from, so we implement a shared secret approach as described here:
+			// https://learn.microsoft.com/en-us/azure/event-grid/security-authentication#using-client-secret-as-a-query-parameter
+			auth := r.Header.Get("authorization")
+			if auth == "" {
+				s.logger.Error("webhook request was missing 'authorization' header")
+				http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+				return
+			}
+			validAuthIdx, valid := s.findValidAuthSecret(auth)
+			if !valid {
+				s.logger.Error("webhook request had invalid 'authorization' header", zap.String("invalid_auth", auth))
 				http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 				return
 			}
+			s.logger.Info("received webhook request with valid 'authorization' header", zap.Int("auth_secret_index", validAuthIdx), zap.String("path", r.URL.Path))
 			next.ServeHTTP(w, r)
 			return
 		}

cmd/server/main.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -87,7 +88,7 @@ func run(args []string) error {
 		azStorageAccount           = fs.String("secret_azure_storage_account", "", "The storage account to authenticate against for blob operations")
 		azSourcePortfolioContainer = fs.String("secret_azure_source_portfolio_container", "", "The container in the storage account where we write raw portfolios to")
 
-		azEventWebhookSecret = fs.String("secret_azure_webhook_secret", "", "The shared secret required for incoming webhooks")
+		azEventWebhookSecrets = fs.String("secret_azure_webhook_secrets", "", "A comma-separated list of shared secrets we'll accept for incoming webhooks")
 
 		runnerConfigLocation   = fs.String("secret_runner_config_location", "", "Location (like 'centralus') where the runner jobs should be executed")
 		runnerConfigConfigPath = fs.String("secret_runner_config_config_path", "", "Config path (like '/configs/dev.conf') where the runner jobs should read their base config from")
@@ -284,7 +285,7 @@ func run(args []string) error {
 
 	eventSrv, err := azevents.NewServer(&azevents.Config{
 		Logger:                      logger,
-		AuthSecret:                  *azEventWebhookSecret,
+		AllowedAuthSecrets:          strings.Split(*azEventWebhookSecrets, ","),
 		Subscription:                *azEventSubscription,
 		ResourceGroup:               *azEventResourceGroup,
 		ProcessedPortfolioTopicName: *azEventProcessedPortfolioTopic,

scripts/run_server.sh

@@ -56,7 +56,7 @@ declare -a FLAGS=(
   "--local_docker_tenant_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .tenant_id)"
   "--local_docker_client_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .client_id)"
   "--local_docker_client_secret=$(echo $LOCAL_DOCKER_CREDS | jq -r .password)"
-  "--secret_azure_webhook_secret=${WEBHOOK_SHARED_SECRET}"
+  "--secret_azure_webhook_secrets=${WEBHOOK_SHARED_SECRET}"
 )
 
 function create_eventgrid_subscription {