Add webhook auth for Event Grid webhooks, and local infra

This PR adds support for a shared secret that gets sent with webhook HTTP requests, so that we can validate they came from Azure. It also adds local support for testing webhooks, which works something like: 1. Developer starts a local server with `--with_public_endpoint=XYZ` 2. Script checks if `local-webhook-XYZ` already exists 3. If not, asks user if they want to create one 4. If yes, waits for server to start (needed for verification to succeed), then creates the webhook
RMI-PACTA · Nov 1, 2023 · 06ac748 · 06ac748
1 parent b96907d
commit 06ac748
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 10 deletions.
diff --git a/azure/azevents/azevents.go b/azure/azevents/azevents.go
@@ -23,6 +23,11 @@ type Config struct {
 	Subscription  string
 	ResourceGroup string
 
+	// AuthSecret is a shared random secret between the event subscription and this
+	// receiver, to prevent random unauthenticated internet requests from triggering
+	// webhooks.
+	AuthSecret string
+
 	ProcessedPortfolioTopicName string
 }
 
@@ -36,6 +41,9 @@ func (c *Config) validate() error {
 	if c.ResourceGroup == "" {
 		return errors.New("no resource group given")
 	}
+	if c.AuthSecret == "" {
+		return errors.New("no auth secret was given")
+	}
 	if c.ProcessedPortfolioTopicName == "" {
 		return errors.New("no resource group given")
 	}
@@ -46,6 +54,8 @@ func (c *Config) validate() error {
 type Server struct {
 	logger *zap.Logger
 
+	authSecret string
+
 	subscription  string
 	resourceGroup string
 	pathToTopic   map[string]string
@@ -58,6 +68,7 @@ func NewServer(cfg *Config) (*Server, error) {
 
 	return &Server{
 		logger:        cfg.Logger,
+		authSecret:    cfg.AuthSecret,
 		subscription:  cfg.Subscription,
 		resourceGroup: cfg.ResourceGroup,
 		pathToTopic: map[string]string{
@@ -76,6 +87,11 @@ func (s *Server) verifyWebhook(next http.Handler) http.Handler {
 		}
 
 		if r.Header.Get("aeg-event-type") != "SubscriptionValidation" {
+			if auth := r.Header.Get("authorization"); auth != s.authSecret {
+				s.logger.Error("missing or invalid auth", zap.String("invalid_auth", auth))
+				http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+				return
+			}
 			next.ServeHTTP(w, r)
 			return
 		}

diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -87,6 +87,8 @@ func run(args []string) error {
 		azStorageAccount           = fs.String("secret_azure_storage_account", "", "The storage account to authenticate against for blob operations")
 		azSourcePortfolioContainer = fs.String("secret_azure_source_portfolio_container", "", "The container in the storage account where we write raw portfolios to")
 
+		azEventWebhookSecret = fs.String("secret_azure_webhook_secret", "", "The shared secret required for incoming webhooks")
+
 		runnerConfigLocation   = fs.String("secret_runner_config_location", "", "Location (like 'centralus') where the runner jobs should be executed")
 		runnerConfigConfigPath = fs.String("secret_runner_config_config_path", "", "Config path (like '/configs/dev.conf') where the runner jobs should read their base config from")
 
@@ -282,6 +284,7 @@ func run(args []string) error {
 
 	eventSrv, err := azevents.NewServer(&azevents.Config{
 		Logger:                      logger,
+		AuthSecret:                  *azEventWebhookSecret,
 		Subscription:                *azEventSubscription,
 		ResourceGroup:               *azEventResourceGroup,
 		ProcessedPortfolioTopicName: *azEventProcessedPortfolioTopic,

diff --git a/scripts/run_server.sh b/scripts/run_server.sh
@@ -35,6 +35,13 @@ fi
 SOPS_DATA="$(sops -d "${ROOT}/secrets/local.enc.json")"
 LOCAL_DOCKER_CREDS="$(echo $SOPS_DATA | jq .localdocker)"
 
+WEBHOOK_CREDS="$(echo $SOPS_DATA | jq .webhook)"
+TOPIC_ID="$(echo $WEBHOOK_CREDS | jq -r .topic_id)"
+WEBHOOK_SHARED_SECRET="$(echo $WEBHOOK_CREDS | jq -r .shared_secret)"
+
+FRP="$(echo $SOPS_DATA | jq .frpc)"
+FRP_ADDR="$(echo $FRP | jq -r .addr)"
+
 FRPC_PID=""
 function cleanup {
   if [[ ! -z "${FRPC_PID}" ]]; then
@@ -45,7 +52,23 @@ function cleanup {
 trap cleanup EXIT
 
 eval set --$OPTS
-declare -a FLAGS=()
+declare -a FLAGS=(
+  "--local_docker_tenant_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .tenant_id)"
+  "--local_docker_client_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .client_id)"
+  "--local_docker_client_secret=$(echo $LOCAL_DOCKER_CREDS | jq -r .password)"
+  "--secret_azure_webhook_secret=${WEBHOOK_SHARED_SECRET}"
+)
+
+function create_eventgrid_subscription {
+  az eventgrid event-subscription create \
+    --name "local-webhook-$1" \
+    --source-resource-id "$TOPIC_ID" \
+    --endpoint-type=webhook \
+    --endpoint="https://$1.${FRP_ADDR}/events/processed_portfolio" \
+    --delivery-attribute-mapping "Authorization static $WEBHOOK_SHARED_SECRET true"
+}
+
+EG_SUB_NAME=""
 while [ ! $# -eq 0 ]
 do
   case "$1" in
@@ -57,8 +80,29 @@ do
         echo 'Error: frpc is not installed, cannot run the FRP client/proxy.' >&2
         exit 1
       fi
-      FRP="$(echo $SOPS_DATA | jq .frpc)"
-      FRP_ADDR="$(echo $FRP | jq -r .addr)"
+      SUB_NAME="$2"
+
+      # Check if they already have
+      set +e # Don't exit on error, this command might fail if the topic doesn't exist
+      az eventgrid event-subscription show \
+        --name "local-webhook-${SUB_NAME}" \
+        --source-resource-id "$TOPIC_ID"
+      SUB_CHECK_EXIT_CODE=$?
+      set -e # Back to exiting on error
+
+      if [[ $SUB_CHECK_EXIT_CODE -ne 0 ]]; then
+        # The check failed, meaning the webhook doesn't exist.
+        # Offer to create it.
+        while true; do
+          read -p "Should we create a webhook subscription for you (y/n)?" yn
+          case $yn in
+            [Yy]* ) EG_SUB_NAME="$SUB_NAME"; break;;
+            [Nn]* ) break;;
+            * ) echo "Please answer yes or no.";;
+          esac
+        done
+      fi
+
       echo "Running FRP proxy at ${FRP_ADDR}..."
       frpc http \
     		--server_addr="$FRP_ADDR" \
@@ -92,10 +136,24 @@ FLAGS+=(
   "--local_dsn=${LOCAL_DSN}"
 )
 
-FLAGS+=(
-  "--local_docker_tenant_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .tenant_id)"
-  "--local_docker_client_id=$(echo $LOCAL_DOCKER_CREDS | jq -r .client_id)"
-  "--local_docker_client_secret=$(echo $LOCAL_DOCKER_CREDS | jq -r .password)"
-)
+if [[ ! -z "$EG_SUB_NAME" ]]; then
+  {
+    # We wait because Event Grid requires validating the subscription, which will fail if we aren't running the validation endpoint.
+    echo "Waiting for server to start before creating EventGrid subscription"
+    sleep 10
+    echo "Creating subscription..."
+
+    set +e # Don't exit on error, we can just alert for this
+  usage error: --delivery-attribute-mapping NAME TYPE [SOURCEFIELD] [VALUE] [ISSECRET]
+    az eventgrid event-subscription create \
+      --name "local-webhook-$EG_SUB_NAME" \
+      --source-resource-id "$TOPIC_ID" \
+      --endpoint-type=webhook \
+      --endpoint="https://${EG_SUB_NAME}.${FRP_ADDR}/events/processed_portfolio" \
+      --delivery-attribute-mapping Authorization static "$WEBHOOK_SHARED_SECRET" true
+    set -e
+  } &
+fi
+
 
 bazel run --run_under="cd $ROOT && " //cmd/server -- "${FLAGS[@]}"
diff --git a/secrets/local.enc.json b/secrets/local.enc.json
@@ -9,6 +9,10 @@
 		"port": "ENC[AES256_GCM,data:d1sh9w==,iv:bmzN6duVQylgRXAchkPavtUsdbNdd+S3f7N73ukFV2A=,tag:4nFCulM5yYdMvTY2CwiJ0g==,type:float]",
 		"token": "ENC[AES256_GCM,data:qe9Hy0tKKrQ7rFzOQuTlDKGDqA2489hvEbjbtYFMuJ3wd5gy,iv:gV+OasDEHtI28TVBFmyxrPjMqEv5J1jpWMAzR1N70MQ=,tag:543oFV21kJj9qBgkyewdIg==,type:str]"
 	},
+	"webhook": {
+		"topic_id": "ENC[AES256_GCM,data:7G0ePwXfG/ePhZu20SI43UOMOTrJ7uS63hAT7vq7yN/ef8WPemEOaQG2/Zs9/4FE01p2eaCntaowE1Sb7FxYnL3SCYN6LyL27wKAESNVW/4uxBqMlhFiNkB/DucahyrA8ko3MIpsT3wqnkMPJScaqhtyBiyv9pLmGrulyaj2DPLyVvnZQxb4D57jVWrRnGZNMik=,iv:Jdnb/9B4MugRcTKpR8oJsb+UksU1FOQnV5Oikut73NM=,tag:ZzJQ9qBWWVDSSzdWQq8LQw==,type:str]",
+		"shared_secret": "ENC[AES256_GCM,data:qzSiBGwCIX4pC8jGB2FKo1bu2EQYXj/T,iv:s94r3EzfnDWvWDXtvpktAnA9Htfd1vcGSH7iF/z6Mxs=,tag:RmhDMwasvOZvccX+QGXBhA==,type:str]"
+	},
 	"sops": {
 		"kms": null,
 		"gcp_kms": null,
@@ -23,8 +27,8 @@
 		],
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2023-10-31T18:08:10Z",
-		"mac": "ENC[AES256_GCM,data:OtzdhIFbPxagE61i0gA871aw8Pkl/PZrwigtX2S3tisOviuYwK0h1QNZN+eKTDXyEbs7fht4ruo4m6EEmZS2nwvgquAUaoA9A9yXARfYFRPQ3D5a21WH8nlN0JMBGoD70VwGjw22Yy2XfG5FbaOnVN2eQ5QpSHaDw1umEiR0rWE=,iv:YZIof2lisX96LDntHnsyDKzQjtlel5SM+Tvpq4C9Sn0=,tag:gEGFEGNB1Fvf1VMKLokMYg==,type:str]",
+		"lastmodified": "2023-11-01T22:30:13Z",
+		"mac": "ENC[AES256_GCM,data:rTQnmHo5AAfJLAEDcnkvP3WI8AvFuDXtajJkrVrsQjccs9gOYR5Zg5w9+Y4pQNuxcHxL/Q3AIGUr/SRMfAyJNsVHOQWdvgbxz0EsxVgc5993kl08uFddL4O4CrZEbfdr8LZEKKIL8fCa5HQHszA1rt2xS/zRJMj6d9+z3ed3O34=,iv:RmJj2I9ki/MxtUN3mYpnOAorSLG0M59gYhJ9Er1szxI=,tag:emPSYpnCz+oPVBznOXt6OA==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.8.1"