Added Host level waited restart logic.

Signed-off-by: utkarshbhatthere <[email protected]>

microceph/api/configs.go

@@ -65,7 +65,7 @@ func cmdConfigsPut(s *state.State, r *http.Request) response.Response {
 	// Restart Daemons on host.
 	daemons := configTable[req.Key].Daemons
 	for i := range daemons {
-		err = ceph.RestartCephDaemon(daemons[i])
+		err = ceph.RestartCephService(daemons[i])
 		if err != nil {
 			return response.SmartError(err)
 		}
@@ -96,7 +96,7 @@ func cmdConfigsDelete(s *state.State, r *http.Request) response.Response {
 	// Restart Daemons on host.
 	daemons := configTable[req.Key].Daemons
 	for i := range daemons {
-		err = ceph.RestartCephDaemon(daemons[i])
+		err = ceph.RestartCephService(daemons[i])
 		if err != nil {
 			return response.SmartError(err)
 		}

microceph/ceph/config.go

@@ -21,10 +21,10 @@ type ConfigExtras struct{
 	Cb      *func(s common.StateInterface, c types.Config)
 }
 
-type ConfigItem map[string]ConfigExtras
+type ConfigTable map[string]ConfigExtras
 
 // Check if certain key is present in the map.
-func (c ConfigItem) isKeyPresent(key string) bool {
+func (c ConfigTable) isKeyPresent(key string) bool {
 	if _, ok := c[key]; !ok {
         return false
     }
@@ -33,15 +33,15 @@ func (c ConfigItem) isKeyPresent(key string) bool {
 }
 
 // Return keys of the given set
-func (c ConfigItem) Keys() (keys []string) {
+func (c ConfigTable) Keys() (keys []string) {
     for k := range c {
         keys = append(keys, k)
     }
     return keys
 }
 
-func GetConfigTable() ConfigItem {
-	return ConfigItem{
+func GetConfigTable() ConfigTable {
+	return ConfigTable{
 		"public_network":  {"global", []string{"mon", "osd"}, "", nil},
 		"cluster_network": {"global", []string{"osd"}, "", nil},
 	}
@@ -56,10 +56,6 @@ type ConfigDump []struct{
 	Value   string
 }
 
-func RestartCephDaemon(daemon string) error {
-	return snapReload(daemon)
-}
-
 func SetConfigItem(s common.StateInterface, c types.Config) error {
 	args := []string{
 		"config",

microceph/ceph/services.go

@@ -5,12 +5,105 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/Rican7/retry"
+	"github.com/Rican7/retry/backoff"
+	"github.com/Rican7/retry/strategy"
 	"github.com/canonical/microcluster/state"
 
 	"github.com/canonical/microceph/microceph/api/types"
 	"github.com/canonical/microceph/microceph/database"
+	"github.com/tidwall/gjson"
 )
 
+type Set map[string]int
+
+func (sub Set) isIn(super Set) bool {
+	flag := true
+
+	// mark flag false if any key from subset is not present in superset.
+	for key := range sub {
+		_, ok := super[key]
+		if !ok {
+			flag = false
+			break // Break the loop.
+		}
+	}
+
+	return flag
+}
+
+// Table to map fetchFunc for workers (daemons) to a service.
+var serviceWorkerTable = map[string](func () (Set, error)) {
+	"osd": getUpOsds,
+	"mon": getMons,
+}
+
+func RestartCephService(service string) error {
+
+	if _, ok := serviceWorkerTable[service]; !ok {
+		return fmt.Errorf("No handler defined for service %s", service)
+	}
+
+	workers, err := serviceWorkerTable[service]()
+	if err != nil {
+		return err
+	}
+
+	// Reload the service.
+	snapReload(service)
+
+	err = retry.Retry(func(i uint) error {
+		iWorkers, err := serviceWorkerTable[service]()
+		if err != nil {
+			return err
+		}
+
+		// All still not up
+		if !workers.isIn(iWorkers) {
+			return fmt.Errorf(
+				"Attempt %d: Workers: %v not all present in %v", i, workers, iWorkers,
+			)
+		}
+		return nil
+	}, strategy.Delay(5), strategy.Limit(10), strategy.Backoff(backoff.Linear(5)))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getMons() (Set, error) {
+	var retval Set
+	output, err := processExec.RunCommand("ceph", "mon", "dump", "-f", "json-pretty")
+	if err != nil {
+		return nil, err
+	}
+
+	// Get a list of mons.
+	mons := gjson.Get(output, "mons.#.name")
+	for _, key := range mons.Array() {
+		retval[key.String()] = 1
+	}
+
+	return retval, nil
+}
+
+func getUpOsds() (Set, error) {
+	var retval Set
+	output, err := processExec.RunCommand("ceph", "osd", "dump", "-f", "json-pretty")
+	if err != nil {
+		return nil, err
+	}
+
+	// Get a list of uuid of osds in up state.
+	upOsds := gjson.Get(output, "osds.#(up==1)#.uuid")
+	for _, element := range upOsds.Array() {
+		retval[element.String()] = 1
+	}
+	return retval, nil
+}
+
 // ListServices retrieves a list of services from the database
 func ListServices(s *state.State) (types.Services, error) {
 	services := types.Services{}

microceph/go.mod

@@ -5,14 +5,15 @@ go 1.18
 require (
 	github.com/canonical/microcluster v0.0.0-20230501200316-dd78e864d2f1
 	github.com/lxc/lxd v0.0.0-20230501200206-976cd2bfee6a
+	github.com/Rican7/retry v0.3.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pborman/uuid v1.2.1
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.2
+	github.com/tidwall/gjson v1.14.4
 )
 
 require (
-	github.com/Rican7/retry v0.3.1 // indirect
 	github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a // indirect
 	github.com/canonical/go-dqlite v1.11.9 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -51,6 +52,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/zitadel/oidc/v2 v2.5.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect

microceph/go.sum

@@ -325,6 +325,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=