[raft] seperate out listen addr and grpc addr passed into dragonboat (#…

…8216) We want to listen on 0.0.0.0:4772 in order for port-forward to work; but we also want the hostname as part of the grpc address when passed into raft.
buildbuddy-io · Jan 21, 2025 · 6e0cb0b · 6e0cb0b
1 parent e921402
commit 6e0cb0b
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 33 deletions.
diff --git a/enterprise/server/raft/cache/cache.go b/enterprise/server/raft/cache/cache.go
@@ -43,9 +43,12 @@ import (
 )
 
 var (
-	rootDirectory           = flag.String("cache.raft.root_directory", "", "The root directory to use for storing cached data.")
-	httpAddr                = flag.String("cache.raft.http_addr", "", "The address to listen for HTTP raft traffic. Ex. '1992'")
-	gRPCAddr                = flag.String("cache.raft.grpc_addr", "", "The address to listen for internal API traffic on. Ex. '1993'")
+	rootDirectory = flag.String("cache.raft.root_directory", "", "The root directory to use for storing cached data.")
+	httpPort      = flag.Int("cache.raft.http_port", 7238, "The address to listen for HTTP raft traffic. Ex. '1992'")
+	gRPCPort      = flag.Int("cache.raft.grpc_port", 4772, "The port to listen for internal Raft API traffic on. Ex. '4772'")
+	hostName      = flag.String("cache.raft.host_name", "", "The hostname of the raft store.")
+	listen        = flag.String("cache.raft.listen", "0.0.0.0", "The interface to listen on (default:0.0.0.0)")
+
 	clearCacheOnStartup     = flag.Bool("cache.raft.clear_cache_on_startup", false, "If set, remove all raft + cache data on start")
 	clearPrevCacheOnStartup = flag.Bool("cache.raft.clear_prev_cache_on_startup", false, "If set, remove all raft + cache data from previous run on start")
 	partitions              = flag.Slice("cache.raft.partitions", []disk.Partition{}, "")
@@ -73,11 +76,12 @@ type Config struct {
 	// Required fields.
 	RootDir string
 
-	// Raft Address
-	HTTPAddr string
+	Hostname string
+
+	ListenAddr string
 
-	// GRPC Address
-	GRPCAddr string
+	HTTPPort int
+	GRPCPort int
 
 	Partitions        []disk.Partition
 	PartitionMappings []disk.PartitionMapping
@@ -92,8 +96,8 @@ type RaftCache struct {
 	env  environment.Env
 	conf *Config
 
-	raftAddress string
-	grpcAddress string
+	raftAddr string
+	grpcAddr string
 
 	registry      registry.NodeRegistry
 	gossipManager interfaces.GossipService
@@ -137,7 +141,7 @@ func clearPrevCache(dir string, currentSubDir string) error {
 }
 
 func Register(env *real_environment.RealEnv) error {
-	if *httpAddr == "" {
+	if *hostName == "" {
 		return nil
 	}
 
@@ -169,8 +173,10 @@ func Register(env *real_environment.RealEnv) error {
 
 	rcConfig := &Config{
 		RootDir:           filepath.Join(*rootDirectory, *subdir),
-		HTTPAddr:          *httpAddr,
-		GRPCAddr:          *gRPCAddr,
+		Hostname:          *hostName,
+		ListenAddr:        *listen,
+		HTTPPort:          *httpPort,
+		GRPCPort:          *gRPCPort,
 		Partitions:        ps,
 		PartitionMappings: *partitionMappings,
 	}
@@ -206,15 +212,16 @@ func NewRaftCache(env environment.Env, conf *Config) (*RaftCache, error) {
 		return nil, err
 	}
 
-	rc.raftAddress = conf.HTTPAddr
-	rc.grpcAddress = conf.GRPCAddr
-
 	if env.GetGossipService() == nil {
 		return nil, status.FailedPreconditionError("raft cache requires gossip be enabled")
 	}
 	rc.gossipManager = env.GetGossipService()
 
-	store, err := store.New(rc.env, conf.RootDir, rc.raftAddress, rc.grpcAddress, rc.conf.Partitions)
+	rc.raftAddr = fmt.Sprintf("%s:%d", conf.Hostname, conf.HTTPPort)
+	rc.grpcAddr = fmt.Sprintf("%s:%d", conf.Hostname, conf.GRPCPort)
+	grpcListeningAddr := fmt.Sprintf("%s:%d", conf.ListenAddr, conf.GRPCPort)
+
+	store, err := store.New(rc.env, conf.RootDir, rc.raftAddr, rc.grpcAddr, grpcListeningAddr, rc.conf.Partitions)
 	if err != nil {
 		return nil, err
 	}
@@ -225,7 +232,7 @@ func NewRaftCache(env environment.Env, conf *Config) (*RaftCache, error) {
 
 	// bring up any clusters that were previously configured, or
 	// bootstrap a new one based on the join params in the config.
-	rc.clusterStarter = bringup.New(rc.grpcAddress, rc.gossipManager, rc.store)
+	rc.clusterStarter = bringup.New(rc.grpcAddr, rc.gossipManager, rc.store)
 	if err := rc.clusterStarter.InitializeClusters(); err != nil {
 		return nil, err
 	}
@@ -254,8 +261,8 @@ func NewRaftCache(env environment.Env, conf *Config) (*RaftCache, error) {
 func (rc *RaftCache) Statusz(ctx context.Context) string {
 	buf := "<pre>"
 	buf += fmt.Sprintf("Root directory: %q\n", rc.conf.RootDir)
-	buf += fmt.Sprintf("Raft (HTTP) addr: %s\n", rc.conf.HTTPAddr)
-	buf += fmt.Sprintf("GRPC addr: %s\n", rc.conf.GRPCAddr)
+	buf += fmt.Sprintf("Raft (HTTP) addr: %s\n", rc.raftAddr)
+	buf += fmt.Sprintf("GRPC addr: %s\n", rc.grpcAddr)
 	buf += fmt.Sprintf("ClusterStarter complete: %t\n", rc.clusterStarter.Done())
 	buf += "</pre>"
 	return buf

diff --git a/enterprise/server/raft/cache/cache_test.go b/enterprise/server/raft/cache/cache_test.go
@@ -84,9 +84,11 @@ func localAddr(t *testing.T) string {
 
 func getCacheConfig(t *testing.T) *raft_cache.Config {
 	return &raft_cache.Config{
-		RootDir:  testfs.MakeTempDir(t),
-		HTTPAddr: localAddr(t),
-		GRPCAddr: localAddr(t),
+		RootDir:    testfs.MakeTempDir(t),
+		Hostname:   "127.0.0.1",
+		ListenAddr: "127.0.0.1",
+		HTTPPort:   testport.FindFree(t),
+		GRPCPort:   testport.FindFree(t),
 	}
 }
 

diff --git a/enterprise/server/raft/store/store.go b/enterprise/server/raft/store/store.go
@@ -168,16 +168,16 @@ func (rc *registryHolder) Create(nhid string, streamConnections uint64, v dbConf
 	return r, nil
 }
 
-func New(env environment.Env, rootDir, raftAddress, grpcAddr string, partitions []disk.Partition) (*Store, error) {
+func New(env environment.Env, rootDir, raftAddr, grpcAddr, grpcListeningAddr string, partitions []disk.Partition) (*Store, error) {
 	rangeCache := rangecache.New()
 	raftListener := listener.NewRaftListener()
 	gossipManager := env.GetGossipService()
-	regHolder := &registryHolder{raftAddress, grpcAddr, gossipManager, nil}
+	regHolder := &registryHolder{raftAddr, grpcAddr, gossipManager, nil}
 	nhc := dbConfig.NodeHostConfig{
 		WALDir:         filepath.Join(rootDir, "wal"),
 		NodeHostDir:    filepath.Join(rootDir, "nodehost"),
 		RTTMillisecond: constants.RTTMillisecond,
-		RaftAddress:    raftAddress,
+		RaftAddress:    raftAddr,
 		Expert: dbConfig.ExpertConfig{
 			NodeRegistryFactory: regHolder,
 		},
@@ -203,10 +203,10 @@ func New(env environment.Env, rootDir, raftAddress, grpcAddr string, partitions
 		return nil, err
 	}
 	leaser := pebble.NewDBLeaser(db)
-	return NewWithArgs(env, rootDir, nodeHost, gossipManager, sender, registry, raftListener, apiClient, grpcAddr, partitions, db, leaser, mc)
+	return NewWithArgs(env, rootDir, nodeHost, gossipManager, sender, registry, raftListener, apiClient, grpcAddr, grpcListeningAddr, partitions, db, leaser, mc)
 }
 
-func NewWithArgs(env environment.Env, rootDir string, nodeHost *dragonboat.NodeHost, gossipManager interfaces.GossipService, sender *sender.Sender, registry registry.NodeRegistry, listener *listener.RaftListener, apiClient *client.APIClient, grpcAddress string, partitions []disk.Partition, db pebble.IPebbleDB, leaser pebble.Leaser, mc *pebble.MetricsCollector) (*Store, error) {
+func NewWithArgs(env environment.Env, rootDir string, nodeHost *dragonboat.NodeHost, gossipManager interfaces.GossipService, sender *sender.Sender, registry registry.NodeRegistry, listener *listener.RaftListener, apiClient *client.APIClient, grpcAddress, grpcListeningAddr string, partitions []disk.Partition, db pebble.IPebbleDB, leaser pebble.Leaser, mc *pebble.MetricsCollector) (*Store, error) {
 	nodeLiveness := nodeliveness.New(env.GetServerContext(), nodeHost.ID(), sender)
 
 	nhLog := log.NamedSubLogger(nodeHost.ID())
@@ -287,7 +287,7 @@ func NewWithArgs(env environment.Env, rootDir string, nodeHost *dragonboat.NodeH
 	grpc_prometheus.Register(s.grpcServer)
 	rfspb.RegisterApiServer(s.grpcServer, s)
 
-	lis, err := net.Listen("tcp", s.grpcAddr)
+	lis, err := net.Listen("tcp", grpcListeningAddr)
 	if err != nil {
 		return nil, err
 	}

diff --git a/enterprise/server/raft/testutil/testutil.go b/enterprise/server/raft/testutil/testutil.go
@@ -121,7 +121,7 @@ func (sf *StoreFactory) RecreateStore(t *testing.T, ts *TestingStore) {
 	require.NoError(t, err)
 	leaser := pebble.NewDBLeaser(db)
 	ts.leaser = leaser
-	store, err := store.NewWithArgs(te, ts.RootDir, nodeHost, ts.gm, s, reg, raftListener, apiClient, ts.GRPCAddress, partitions, db, leaser, mc)
+	store, err := store.NewWithArgs(te, ts.RootDir, nodeHost, ts.gm, s, reg, raftListener, apiClient, ts.GRPCAddress, ts.GRPCAddress, partitions, db, leaser, mc)
 	require.NoError(t, err)
 	require.NotNil(t, store)
 	store.Start()

diff --git a/tools/goreman/procfiles/Procfile.raft b/tools/goreman/procfiles/Procfile.raft
@@ -1,7 +1,9 @@
-app1: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9901 --grpc_port=1970 --monitoring_port=9101 --internal_grpc_port=7701 --disable_telemetry --cache.raft.root_directory=/tmp/rnode1 --gossip.listen_addr=127.0.0.1:9201 --gossip.node_name=app1 --cache.raft.http_addr=127.0.0.1:9301 --cache.raft.grpc_addr=127.0.0.1:9401 --auto_migrate_db=false
+app1: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9901 --grpc_port=1970 --monitoring_port=9101 --internal_grpc_port=7701 --disable_telemetry --cache.raft.root_directory=/tmp/rnode1 --gossip.listen_addr=127.0.0.1:9201 --gossip.node_name=app1 --cache.raft.host_name=127.0.0.1 --cache.raft.http_port=9301 --cache.raft.grpc_port=9401 --auto_migrate_db=false
 
-app2: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9902 --grpc_port=1971 --monitoring_port=9102 --internal_grpc_port=7702 --disable_telemetry --cache.raft.root_directory=/tmp/rnode2 --gossip.listen_addr=127.0.0.1:9202 --gossip.node_name=app2  --cache.raft.http_addr=127.0.0.1:9302 --cache.raft.grpc_addr=127.0.0.1:9402 --auto_migrate_db=false
+app2: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9902 --grpc_port=1971 --monitoring_port=9102 --internal_grpc_port=7702 --disable_telemetry --cache.raft.root_directory=/tmp/rnode2 --gossip.listen_addr=127.0.0.1:9202 --gossip.node_name=app2 --cache.raft.host_name=127.0.0.1 --cache.raft.http_port=9302 --cache.raft.grpc_port=9402 --auto_migrate_db=false
 
-app3: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9903 --grpc_port=1972 --monitoring_port=9103 --internal_grpc_port=7703 --disable_telemetry --cache.raft.root_directory=/tmp/rnode3 --gossip.listen_addr=127.0.0.1:9203 --gossip.node_name=app3 --cache.raft.http_addr=127.0.0.1:9303 --cache.raft.grpc_addr=127.0.0.1:9403 --auto_migrate_db=false
+app3: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9903 --grpc_port=1972 --monitoring_port=9103 --internal_grpc_port=7703 --disable_telemetry --cache.raft.root_directory=/tmp/rnode3 --gossip.listen_addr=127.0.0.1:9203 --gossip.node_name=app3 --cache.raft.host_name=127.0.0.1 --cache.raft.http_port=9303 --cache.raft.grpc_port=9403 --auto_migrate_db=false
 
-app4: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9904 --grpc_port=1973 --monitoring_port=9104 --internal_grpc_port=7704 --disable_telemetry --cache.raft.root_directory=/tmp/rnode4 --gossip.listen_addr=127.0.0.1:9204 --gossip.node_name=app4 --cache.raft.http_addr=127.0.0.1:9304 --cache.raft.grpc_addr=127.0.0.1:9404 --auto_migrate_db=false
+app4: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9904 --grpc_port=1973 --monitoring_port=9104 --internal_grpc_port=7704 --disable_telemetry --cache.raft.root_directory=/tmp/rnode4 --gossip.listen_addr=127.0.0.1:9204 --gossip.node_name=app4 --cache.raft.host_name=127.0.0.1 --cache.raft.http_port=9304 --cache.raft.grpc_port=9404 --auto_migrate_db=false
+
+app5: bazel run enterprise/server -- --config_file=enterprise/config/buildbuddy.raft.yaml --telemetry_port=9905 --grpc_port=1974 --monitoring_port=9105 --internal_grpc_port=7705 --disable_telemetry --cache.raft.root_directory=/tmp/rnode5 --gossip.listen_addr=127.0.0.1:9205 --gossip.node_name=app5 --cache.raft.host_name=127.0.0.1 --cache.raft.http_port=9305 --cache.raft.grpc_port=9405 --auto_migrate_db=false