feat: Add cannon support to docker-compose (#396)

* feat: Add cannon support to docker-compose

* feat: Add cannon support to docker-compose

* fix

* refactor: Comment out unused authorization settings

* style: Comment out unnecessary environment variables

cmd/cannon.go

@@ -14,6 +14,84 @@ var (
 	cannonCfgFile string
 )
 
+type CannonOverride struct {
+	FlagHelper func(cmd *cobra.Command)
+	Setter     func(cmd *cobra.Command, overrides *cannon.Override) error
+}
+
+type CannonOverrideConfig struct {
+	FlagName     string
+	EnvName      string
+	Description  string
+	OverrideFunc func(val string, overrides *cannon.Override)
+}
+
+func createCannonOverride(config CannonOverrideConfig) CannonOverride {
+	return CannonOverride{
+		FlagHelper: func(cmd *cobra.Command) {
+			cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+		},
+		Setter: func(cmd *cobra.Command, overrides *cannon.Override) error {
+			val := ""
+
+			if cmd.Flags().Changed(config.FlagName) {
+				val = cmd.Flags().Lookup(config.FlagName).Value.String()
+			}
+
+			if os.Getenv(config.EnvName) != "" {
+				val = os.Getenv(config.EnvName)
+			}
+
+			if val == "" {
+				return nil
+			}
+
+			config.OverrideFunc(val, overrides)
+
+			return nil
+		},
+	}
+}
+
+var CannonOverrides = []CannonOverride{
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-xatu-output-authorization",
+		EnvName:     "CANNON_XATU_OUTPUT_AUTHORIZATION",
+		Description: "sets the authorization secret for all xatu outputs",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.XatuOutputAuth.Enabled = true
+			overrides.XatuOutputAuth.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-xatu-coordinator-authorization",
+		EnvName:     "CANNON_XATU_COORDINATOR_AUTHORIZATION",
+		Description: "sets the authorization secret for the xatu coordinator",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.XatuCoordinatorAuth.Enabled = true
+			overrides.XatuCoordinatorAuth.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-beacon-node-url",
+		EnvName:     "CANNON_BEACON_NODE_URL",
+		Description: "sets the beacon node url",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.BeaconNodeURL.Enabled = true
+			overrides.BeaconNodeURL.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-beacon-node-authorization-header",
+		EnvName:     "CANNON_BEACON_NODE_AUTHORIZATION_HEADER",
+		Description: "sets the beacon node authorization header",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.BeaconNodeAuthorizationHeader.Enabled = true
+			overrides.BeaconNodeAuthorizationHeader.Value = val
+		},
+	}),
+}
+
 // cannonCmd represents the cannon command
 var cannonCmd = &cobra.Command{
 	Use:   "cannon",
@@ -32,7 +110,15 @@ var cannonCmd = &cobra.Command{
 
 		log.WithField("location", cannonCfgFile).Info("Loaded config")
 
-		cannon, err := cannon.New(cmd.Context(), log, config)
+		overrides := &cannon.Override{}
+
+		for _, override := range CannonOverrides {
+			if errr := override.Setter(cmd, overrides); errr != nil {
+				log.Fatal(errr)
+			}
+		}
+
+		cannon, err := cannon.New(cmd.Context(), log, config, overrides)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -49,6 +135,10 @@ func init() {
 	rootCmd.AddCommand(cannonCmd)
 
 	cannonCmd.Flags().StringVar(&cannonCfgFile, "config", "cannon.yaml", "config file (default is cannon.yaml)")
+
+	for _, override := range CannonOverrides {
+		override.FlagHelper(cannonCmd)
+	}
 }
 
 func loadcannonConfigFromFile(file string) (*cannon.Config, error) {

cmd/server.go

@@ -13,12 +13,78 @@ var (
 	serverCfgFile string
 )
 
+type ServerOverride struct {
+	FlagHelper func(cmd *cobra.Command)
+	Setter     func(cmd *cobra.Command, overrides *server.Override) error
+}
+
+type ServerOverrideConfig struct {
+	FlagName     string
+	EnvName      string
+	Description  string
+	OverrideFunc func(val string, overrides *server.Override)
+}
+
+func createServerOverride(config ServerOverrideConfig) ServerOverride {
+	return ServerOverride{
+		FlagHelper: func(cmd *cobra.Command) {
+			cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+		},
+		Setter: func(cmd *cobra.Command, overrides *server.Override) error {
+			val := ""
+
+			if cmd.Flags().Changed(config.FlagName) {
+				val = cmd.Flags().Lookup(config.FlagName).Value.String()
+			}
+
+			if os.Getenv(config.EnvName) != "" {
+				val = os.Getenv(config.EnvName)
+			}
+
+			if val == "" {
+				return nil
+			}
+
+			config.OverrideFunc(val, overrides)
+
+			return nil
+		},
+	}
+}
+
+var ServerOverrides = []ServerOverride{
+	createServerOverride(ServerOverrideConfig{
+		FlagName:    "server-event-ingester-basic-auth-username",
+		EnvName:     "SERVER_EVENT_INGESTER_BASIC_AUTH_USERNAME",
+		Description: "sets the basic auth username for the event ingester",
+		OverrideFunc: func(val string, overrides *server.Override) {
+			overrides.EventIngesterBasicAuth.Username = val
+		},
+	}),
+	createServerOverride(ServerOverrideConfig{
+		FlagName:    "server-event-ingester-basic-auth-password",
+		EnvName:     "SERVER_EVENT_INGESTER_BASIC_AUTH_PASSWORD",
+		Description: "sets the basic auth password for the event ingester",
+		OverrideFunc: func(val string, overrides *server.Override) {
+			overrides.EventIngesterBasicAuth.Password = val
+		},
+	}),
+	createServerOverride(ServerOverrideConfig{
+		FlagName:    "server-coordinator-auth-secret",
+		EnvName:     "SERVER_COORDINATOR_AUTH_SECRET",
+		Description: "sets the auth secret for the coordinator",
+		OverrideFunc: func(val string, overrides *server.Override) {
+			overrides.CoordinatorAuth.AuthSecret = val
+		},
+	}),
+}
+
 // serverCmd represents the server command
 var serverCmd = &cobra.Command{
 	Use:   "server",
 	Short: "Runs Xatu in Server mode.",
 	Long: `Runs Xatu in Server mode, which means it will listen to gRPC requests from
-	Xatu Sentry nodes and forward the data on to the configured sinks.`,
+	Xatu modules and forward the data on to the configured sinks.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		initCommon()
 
@@ -31,7 +97,15 @@ var serverCmd = &cobra.Command{
 
 		log.WithField("location", serverCfgFile).Info("Loaded config")
 
-		server, err := server.NewXatu(cmd.Context(), log, config)
+		o := &server.Override{}
+
+		for _, override := range ServerOverrides {
+			if errr := override.Setter(cmd, o); errr != nil {
+				log.Fatal(errr)
+			}
+		}
+
+		server, err := server.NewXatu(cmd.Context(), log, config, o)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -48,6 +122,10 @@ func init() {
 	rootCmd.AddCommand(serverCmd)
 
 	serverCmd.Flags().StringVar(&serverCfgFile, "config", "server.yaml", "config file (default is server.yaml)")
+
+	for _, override := range ServerOverrides {
+		override.FlagHelper(serverCmd)
+	}
 }
 
 func loadServerConfigFromFile(file string) (*server.Config, error) {

deploy/local/docker-compose/xatu-cannon.yaml

@@ -0,0 +1,80 @@
+logging: "info" # panic,fatal,warn,info,debug,trace
+metricsAddr: ":9097"
+pprofAddr: ":6061" # optional. if supplied it enables pprof server
+
+name: xatu-cannon
+
+# derivers:
+#   attesterSlashing:
+#     enabled: false
+#   blsToExecutionChange:
+#     enabled: false
+#   deposit:
+#     enabled: false
+#   withdrawal:
+#     enabled: false
+#   executionTransaction:
+#     enabled: false
+#   proposerSlashing:
+#     enabled: false
+#   voluntaryExit:
+#     enabled: false
+#   beaconBlock:
+#     enabled: true
+#   beaconBlobSidecar:
+#     enabled: false
+#   proposerDuty:
+#     enabled: true
+#   elaboratedAttestation:
+#     enabled: false
+#   beaconValidators:
+#     enabled: false
+#   elaboratedAttestation:
+#     enabled: false
+#   beaconCommittee:
+#     enabled: false
+
+  # blockClassification:
+  #   enabled: false
+
+
+
+# Better to use a NTP server close eg.
+#   time.aws.com - AWS
+#   time.windows.com - Azure
+#   time.google.com - GCP
+#   pool.ntp.org - https://www.pool.ntp.org/zone/@
+ntpServer: time.google.com
+
+tracing:
+  enabled: true
+  endpoint: tempo:4318
+  insecure: true
+  sampling:
+    rate: 0.1
+
+ethereum:
+  beaconNodeAddress: http://SET_ME:5052
+  # overrideNetworkName: mainnet
+  # blockCacheSize: 10000
+  # blockCacheTtl: 3h
+  # blockPreloadWorkers: 25
+
+coordinator:
+  address: xatu-server:8080
+  headers:
+    Authorization: "Bearer SET_ME"
+
+outputs:
+  - name: xatu
+    type: xatu
+    config:
+      address: xatu-server:8080
+      tls: false
+      maxQueueSize: 51200
+      batchTimeout: 0.5s
+      exportTimeout: 30s
+      maxExportBatchSize: 32
+      workers: 50
+      headers:
+        Authorization: "Bearer SET_ME"

deploy/local/docker-compose/xatu-server.yaml

@@ -38,7 +38,7 @@ services:
     enabled: true # requires persistence to be enabled
     auth:
       enabled: false
-      secret: "super-secret-token"
+      # secret: "super-secret-token"
     nodeRecord:
       maxQueueSize: 51200
       batchTimeout: 5s
@@ -48,27 +48,27 @@ services:
     enabled: true
     clientNameSalt: "example_salt"
     authorization:
-      enabled: false
-      groups:
-        default:
-          obscureClientNames: true
-          users:
-            user_a:
-              password: password
-            shane:
-              password: warne
-        restricted:
-          eventFilter:
-            eventNames:
-              - BEACON_API_ETH_V2_BEACON_BLOCK_V2
-              - BEACON_API_ETH_V1_EVENTS_BLOB_SIDECAR
-              - BEACON_API_ETH_V1_EVENTS_BLOCK_V2
-              - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG_V2
-              - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT_V2
-              - BEACON_API_ETH_V1_EVENTS_HEAD_V2
-          users:
-            user_b:
-              password: password
+      # enabled: false
+      # groups:
+      #   default:
+      #     obscureClientNames: true
+      #     users:
+      #       user_a:
+      #         password: password
+      #       shane:
+      #         password: warne
+      #   restricted:
+      #     eventFilter:
+      #       eventNames:
+      #         - BEACON_API_ETH_V2_BEACON_BLOCK_V2
+      #         - BEACON_API_ETH_V1_EVENTS_BLOB_SIDECAR
+      #         - BEACON_API_ETH_V1_EVENTS_BLOCK_V2
+      #         - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG_V2
+      #         - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT_V2
+      #         - BEACON_API_ETH_V1_EVENTS_HEAD_V2
+      #     users:
+      #       user_b:
+      #         password: password
     outputs:
     - name: general
       type: http