Add --mon-ip parameter to specify desired IP for monitor bootstrap

Signed-off-by: Utkarsh Bhatt <[email protected]>
canonical · Nov 3, 2023 · 6144a18 · 6144a18
1 parent 895c58a
commit 6144a18
Show file tree

Hide file tree

Showing 13 changed files with 362 additions and 57 deletions.
diff --git a/microceph/api/services.go b/microceph/api/services.go
@@ -22,6 +22,7 @@ var servicesCmd = rest.Endpoint{
 	Path: "services",
 
 	Get: rest.EndpointAction{Handler: cmdServicesGet, ProxyTarget: true},
+	Put: rest.EndpointAction{Handler: cmdServicesPut, ProxyTarget: false},
 }
 
 func cmdServicesGet(s *state.State, r *http.Request) response.Response {
@@ -33,6 +34,24 @@ func cmdServicesGet(s *state.State, r *http.Request) response.Response {
 	return response.SyncResponse(true, services)
 }
 
+// cmdServicesPut bootstraps the internal ceph cluster.
+func cmdServicesPut(s *state.State, r *http.Request) response.Response {
+	var data types.Bootstrap
+
+	err := json.NewDecoder(r.Body).Decode(&data)
+	if err != nil {
+		logger.Errorf("Failed decoding ceph bootstrap request: %v", err)
+		return response.InternalError(err)
+	}
+
+	err = ceph.Bootstrap(common.CephState{State: s}, data)
+	if err != nil {
+		return response.SyncResponse(false, err)
+	}
+
+	return response.EmptySyncResponse
+}
+
 // Service endpoints.
 var monServiceCmd = rest.Endpoint{
 	Path:   "services/mon",

diff --git a/microceph/api/types/services.go b/microceph/api/types/services.go
@@ -26,3 +26,9 @@ type RGWService struct {
 	Port    int  `json:"port" yaml:"port"`
 	Enabled bool `json:"enabled" yaml:"enabled"`
 }
+
+// Bootstrap holds the parameters required for bootstrapping the ceph cluster.
+type Bootstrap struct {
+	MonIp  string `json:"monip" yaml:"monip"`
+	PubNet string `json:"pubnet" yaml:"pubnet"`
+}
diff --git a/microceph/ceph/bootstrap.go b/microceph/ceph/bootstrap.go
@@ -10,32 +10,40 @@ import (
 
 	"github.com/pborman/uuid"
 
+	apiTypes "github.com/canonical/microceph/microceph/api/types"
 	"github.com/canonical/microceph/microceph/common"
 	"github.com/canonical/microceph/microceph/database"
 )
 
 // Bootstrap will initialize a new Ceph deployment.
-func Bootstrap(s common.StateInterface) error {
+func Bootstrap(s common.StateInterface, data apiTypes.Bootstrap) error {
 	pathConsts := common.GetPathConst()
 	pathFileMode := common.GetPathFileMode()
 
 	// Create our various paths.
 	for path, perm := range pathFileMode {
 		err := os.MkdirAll(path, perm)
 		if err != nil {
-			return fmt.Errorf("Unable to create %q: %w", path, err)
+			return fmt.Errorf("unable to create %q: %w", path, err)
 		}
 	}
 
 	// Generate a new FSID.
 	fsid := uuid.NewRandom().String()
 	conf := newCephConfig(pathConsts.ConfPath)
-	err := conf.WriteConfig(
+	pubNet, err := common.Network.FindNetworkAddress(data.MonIp)
+	if err != nil {
+		return nil
+	}
+
+	err = conf.WriteConfig(
 		map[string]any{
-			"fsid":     fsid,
-			"runDir":   pathConsts.RunPath,
-			"monitors": s.ClusterState().Address().Hostname(),
-			"addr":     s.ClusterState().Address().Hostname(),
+			"fsid":   fsid,
+			"runDir": pathConsts.RunPath,
+			// First monitor bootstrap IP as passed to microcluster.
+			"monitors": data.MonIp,
+			"addr":     data.MonIp,
+			"pubnet":   pubNet,
 		},
 		0644,
 	)
@@ -55,7 +63,7 @@ func Bootstrap(s common.StateInterface) error {
 		return fmt.Errorf("Failed parsing admin keyring: %w", err)
 	}
 
-	err = createMonMap(s, path, fsid)
+	err = createMonMap(s, path, fsid, data.MonIp)
 	if err != nil {
 		return err
 	}
@@ -86,7 +94,7 @@ func Bootstrap(s common.StateInterface) error {
 	}
 
 	// Update the database.
-	err = updateDatabase(s, fsid, adminKey)
+	err = populateDatabase(s, fsid, adminKey, data)
 	if err != nil {
 		return err
 	}
@@ -105,7 +113,7 @@ func Bootstrap(s common.StateInterface) error {
 	// Re-generate the configuration from the database.
 	err = UpdateConfig(s)
 	if err != nil {
-		return fmt.Errorf("Failed to re-generate the configuration: %w", err)
+		return fmt.Errorf("failed to re-generate the configuration: %w", err)
 	}
 
 	return nil
@@ -137,16 +145,16 @@ func createKeyrings(confPath string) (string, error) {
 	return path, nil
 }
 
-func createMonMap(s common.StateInterface, path string, fsid string) error {
+func createMonMap(s common.StateInterface, path string, fsid string, address string) error {
 	// Generate initial monitor map.
 	err := genMonmap(filepath.Join(path, "mon.map"), fsid)
 	if err != nil {
-		return fmt.Errorf("Failed to generate monitor map: %w", err)
+		return fmt.Errorf("failed to generate monitor map: %w", err)
 	}
 
-	err = addMonmap(filepath.Join(path, "mon.map"), s.ClusterState().Name(), s.ClusterState().Address().Hostname())
+	err = addMonmap(filepath.Join(path, "mon.map"), s.ClusterState().Name(), address)
 	if err != nil {
-		return fmt.Errorf("Failed to add monitor map: %w", err)
+		return fmt.Errorf("failed to add monitor map: %w", err)
 	}
 
 	return nil
@@ -194,36 +202,48 @@ func initMgr(s common.StateInterface, dataPath string) error {
 	return nil
 }
 
-func updateDatabase(s common.StateInterface, fsid string, adminKey string) error {
+// populateDatabase injects the bootstrap entries to the internal database.
+func populateDatabase(s common.StateInterface, fsid string, adminKey string, data apiTypes.Bootstrap) error {
 	if s.ClusterState().Database == nil {
 		return fmt.Errorf("no database")
 	}
 	err := s.ClusterState().Database.Transaction(s.ClusterState().Context, func(ctx context.Context, tx *sql.Tx) error {
 		// Record the roles.
 		_, err := database.CreateService(ctx, tx, database.Service{Member: s.ClusterState().Name(), Service: "mon"})
 		if err != nil {
-			return fmt.Errorf("Failed to record role: %w", err)
+			return fmt.Errorf("failed to record role: %w", err)
 		}
 
 		_, err = database.CreateService(ctx, tx, database.Service{Member: s.ClusterState().Name(), Service: "mgr"})
 		if err != nil {
-			return fmt.Errorf("Failed to record role: %w", err)
+			return fmt.Errorf("failed to record role: %w", err)
 		}
 
 		_, err = database.CreateService(ctx, tx, database.Service{Member: s.ClusterState().Name(), Service: "mds"})
 		if err != nil {
-			return fmt.Errorf("Failed to record role: %w", err)
+			return fmt.Errorf("failed to record role: %w", err)
 		}
 
 		// Record the configuration.
 		_, err = database.CreateConfigItem(ctx, tx, database.ConfigItem{Key: "fsid", Value: fsid})
 		if err != nil {
-			return fmt.Errorf("Failed to record fsid: %w", err)
+			return fmt.Errorf("failed to record fsid: %w", err)
 		}
 
 		_, err = database.CreateConfigItem(ctx, tx, database.ConfigItem{Key: "keyring.client.admin", Value: adminKey})
 		if err != nil {
-			return fmt.Errorf("Failed to record keyring: %w", err)
+			return fmt.Errorf("failed to record keyring: %w", err)
+		}
+
+		key := fmt.Sprintf("mon.host.%s", s.ClusterState().Name())
+		_, err = database.CreateConfigItem(ctx, tx, database.ConfigItem{Key: key, Value: data.MonIp})
+		if err != nil {
+			return fmt.Errorf("failed to record mon host: %w", err)
+		}
+
+		_, err = database.CreateConfigItem(ctx, tx, database.ConfigItem{Key: "public_network", Value: data.PubNet})
+		if err != nil {
+			return fmt.Errorf("failed to public_network: %w", err)
 		}
 
 		return nil

diff --git a/microceph/ceph/bootstrap_test.go b/microceph/ceph/bootstrap_test.go
@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/canonical/lxd/shared/api"
+	"github.com/canonical/microceph/microceph/api/types"
+	"github.com/canonical/microceph/microceph/common"
 	"github.com/canonical/microceph/microceph/mocks"
 	"github.com/canonical/microcluster/state"
 	"github.com/stretchr/testify/assert"
@@ -56,6 +58,11 @@ func addEnableMsgr2Expectations(r *mocks.Runner) {
 	r.On("RunCommand", cmdAny("snapctl", 3)...).Return("ok", nil).Once()
 }
 
+// Expect: mock
+func addNetworkExpectations(nw *mocks.NetworkIntf, s common.StateInterface) {
+	nw.On("FindNetworkAddress", "1.1.1.1").Return("1.1.1.1/24", nil)
+}
+
 func (s *bootstrapSuite) SetupTest() {
 
 	s.baseSuite.SetupTest()
@@ -78,16 +85,20 @@ func (s *bootstrapSuite) SetupTest() {
 // Test a bootstrap run, mocking subprocess calls but without a live database
 func (s *bootstrapSuite) TestBootstrap() {
 	r := mocks.NewRunner(s.T())
+	nw := mocks.NewNetworkIntf(s.T())
+
 	addCreateKeyringExpectations(r)
 	addCreateMonMapExpectations(r)
 	addInitMonExpectations(r)
 	addInitMgrExpectations(r)
 	addInitMdsExpectations(r)
 	addEnableMsgr2Expectations(r)
+	addNetworkExpectations(nw, s.TestStateInterface)
 
 	processExec = r
+	common.Network = nw
 
-	err := Bootstrap(s.TestStateInterface)
+	err := Bootstrap(s.TestStateInterface, types.Bootstrap{MonIp: "1.1.1.1", PubNet: "1.1.1.1/24"})
 
 	// we expect a missing database error
 	assert.EqualError(s.T(), err, "no database")

diff --git a/microceph/ceph/config.go b/microceph/ceph/config.go
@@ -170,20 +170,13 @@ func UpdateConfig(s common.StateInterface) error {
 	// Get the configuration and servers.
 	var err error
 	var configItems []database.ConfigItem
-	var monitors []database.Service
 
 	err = s.ClusterState().Database.Transaction(s.ClusterState().Context, func(ctx context.Context, tx *sql.Tx) error {
 		configItems, err = database.GetConfigItems(ctx, tx)
 		if err != nil {
 			return err
 		}
 
-		serviceName := "mon"
-		monitors, err = database.GetServices(ctx, tx, database.ServiceFilter{Service: &serviceName})
-		if err != nil {
-			return err
-		}
-
 		return nil
 	})
 	if err != nil {
@@ -195,30 +188,28 @@ func UpdateConfig(s common.StateInterface) error {
 		config[item.Key] = item.Value
 	}
 
-	monitorAddresses := make([]string, len(monitors))
-	remotes := s.ClusterState().Remotes().RemotesByName()
-	for i, monitor := range monitors {
-		remote, ok := remotes[monitor.Member]
-		if !ok {
-			continue
-		}
-
-		monitorAddresses[i] = remote.Address.Addr().String()
-	}
-
+	// REF: https://docs.ceph.com/en/quincy/rados/configuration/network-config-ref/#ceph-daemons
+	// The mon host configuration option only needs to be sufficiently up to date such that a
+	// client can reach one monitor that is currently online.
+	monitorAddresses := getMonitorAddresses(config)
 	conf := newCephConfig(confPath)
-	address := s.ClusterState().Address().Hostname()
+	address, err := common.Network.FindIpOnSubnet(config["public_network"])
+	if err != nil {
+		return fmt.Errorf("failed to record mon host: %w", err)
+	}
 	clientConfig, err := GetClientConfigForHost(s, s.ClusterState().Name())
 	if err != nil {
 		logger.Errorf("Failed to pull Client Configurations: %v", err)
 		return err
 	}
 
+	// Populate Template
 	err = conf.WriteConfig(
 		map[string]any{
 			"fsid":                config["fsid"],
 			"runDir":              runPath,
 			"monitors":            strings.Join(monitorAddresses, ","),
+			"pubnet":              config["public_network"],
 			"addr":                address,
 			"ipv4":                strings.Contains(address, "."),
 			"ipv6":                strings.Contains(address, ":"),
@@ -249,3 +240,14 @@ func UpdateConfig(s common.StateInterface) error {
 
 	return nil
 }
+
+// getMonitorAddresses scans a provided config key/value map and returns a list of mon hosts found.
+func getMonitorAddresses(configs map[string]string) []string {
+	monHosts := []string{}
+	for k, v := range configs {
+		if strings.Contains(k, "mon.host.") {
+			monHosts = append(monHosts, v)
+		}
+	}
+	return monHosts
+}
diff --git a/microceph/ceph/configwriter.go b/microceph/ceph/configwriter.go
@@ -49,6 +49,7 @@ fsid = {{.fsid}}
 mon host = {{.monitors}}
 auth allow insecure global id reclaim = false
 public addr = {{.addr}}
+public_network = {{.pubnet}}
 ms bind ipv4 = {{.ipv4}}
 ms bind ipv6 = {{.ipv6}}
 

diff --git a/microceph/ceph/join.go b/microceph/ceph/join.go
@@ -103,12 +103,46 @@ func Join(s common.StateInterface) error {
 	}
 
 	// Update the database.
-	err = s.ClusterState().Database.Transaction(s.ClusterState().Context, func(ctx context.Context, tx *sql.Tx) error {
+	err = updateDatabasePostJoin(s, services)
+	if err != nil {
+		return fmt.Errorf("failed to update DB post join: %w", err)
+	}
+
+	// Start OSD service.
+	err = snapStart("osd", true)
+	if err != nil {
+		return fmt.Errorf("failed to start OSD service: %w", err)
+	}
+
+	return nil
+}
+
+func updateDatabasePostJoin(s common.StateInterface, services []string) error {
+	err := s.ClusterState().Database.Transaction(s.ClusterState().Context, func(ctx context.Context, tx *sql.Tx) error {
 		// Record the roles.
 		for _, service := range services {
 			_, err := database.CreateService(ctx, tx, database.Service{Member: s.ClusterState().Name(), Service: service})
 			if err != nil {
-				return fmt.Errorf("Failed to record role: %w", err)
+				return fmt.Errorf("failed to record role: %w", err)
+			}
+
+			if service == "mon" {
+				// Fetch public network
+				configItem, err := database.GetConfigItem(ctx, tx, "public_network")
+				if err != nil {
+					return err
+				}
+
+				monHost, err := common.Network.FindIpOnSubnet(configItem.Value)
+				if err != nil {
+					return fmt.Errorf("failed to record mon host: %w", err)
+				}
+
+				key := fmt.Sprintf("mon.host.%s", s.ClusterState().Name())
+				_, err = database.CreateConfigItem(ctx, tx, database.ConfigItem{Key: key, Value: monHost})
+				if err != nil {
+					return fmt.Errorf("failed to record mon host: %w", err)
+				}
 			}
 		}
 
@@ -118,11 +152,5 @@ func Join(s common.StateInterface) error {
 		return err
 	}
 
-	// Start OSD service.
-	err = snapStart("osd", true)
-	if err != nil {
-		return fmt.Errorf("Failed to start OSD service: %w", err)
-	}
-
 	return nil
 }