Added testing and fixed issues

Signed-off-by: utkarshbhatthere <[email protected]>

.github/workflows/tests.yml

@@ -130,6 +130,36 @@ jobs:
           s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl put -P ~/test.txt s3://testbucket
           curl -s http://localhost/testbucket/test.txt | grep -F hello-radosgw
 
+      - name: Test Cluster Config
+        run: |
+          set -eux
+          cip=$(ip -4 -j route | jq -r '.[] | select(.dst | contains("default")) | .prefsrc' | tr -d '[:space:]')
+          
+          # pre config set timestamp for service age
+          ts=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2)
+
+          # set config
+          sudo microceph cluster config set cluster_network $cip/8 --wait
+
+          # post config set timestamp for service age
+          ts2=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2)
+
+          # Check config output
+          output=$(sudo microceph cluster config get cluster_network | grep -cim1 'cluster_network')
+          if [[ $output -lt 1 ]] ; then echo "config check failed: $output"; exit 1; fi
+
+          # Check service restarted
+          if [ $ts2 -lt $ts ]; then echo "config check failed: TS1: $ts2 TS2: $ts3"; exit 1; fi
+
+          # reset config
+          sudo microceph cluster config reset cluster_network --wait
+
+          # post config reset timestamp for service age
+          ts3=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2)
+
+          # Check service restarted
+          if [ $ts3 -lt $ts2 ]; then echo "config check failed: TS2: $ts2 TS3: $ts3"; exit 1; fi
+
       - name: Upload artifacts
         if: always()
         uses: actions/upload-artifact@v3

microceph/api/configs.go

@@ -23,17 +23,21 @@ var configsCmd = rest.Endpoint{
 }
 
 func cmdConfigsGet(s *state.State, r *http.Request) response.Response {
+	var err error
 	var req types.Config
+	var configs types.Configs
 
-	err := json.NewDecoder(r.Body).Decode(&req)
+	err = json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
 		return response.InternalError(err)
 	}
 
-	// Fetch configs.
-	configs, err := ceph.ListConfigs()
-	if err != nil {
-		return response.InternalError(err)
+	// If a valid key string is passed, fetch that key.
+	if len(req.Key) > 0 {
+		configs, err = ceph.GetConfigItem(req)
+	} else {
+		// Fetch all configs.
+		configs, err = ceph.ListConfigs()
 	}
 
 	return response.SyncResponse(true, configs)
@@ -55,12 +59,8 @@ func cmdConfigsPut(s *state.State, r *http.Request) response.Response {
 	}
 
 	services := configTable[req.Key].Daemons
-	go func() {
-		client.SendRestartRequestToClusterMembers(s, r, services)
-		// Restart Daemons on host.
-		ceph.RestartCephServices(services)
-	}()
-
+	client.ConfigChangeRefresh(s, services, req.Wait)
+
 	return response.EmptySyncResponse
 }
 
@@ -80,11 +80,7 @@ func cmdConfigsDelete(s *state.State, r *http.Request) response.Response {
 	}
 
 	services := configTable[req.Key].Daemons
-	go func() {
-		client.SendRestartRequestToClusterMembers(s, r, services)
-		// Restart Daemons on host.
-		ceph.RestartCephServices(services)
-	}()
+	client.ConfigChangeRefresh(s, services, req.Wait)
 
 	return response.EmptySyncResponse
 }

microceph/api/services.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
 
 	"github.com/canonical/microceph/microceph/api/types"
@@ -43,15 +42,15 @@ func cmdRestartServicePost(s *state.State, r *http.Request) response.Response {
 
 	err := json.NewDecoder(r.Body).Decode(&services)
 	if err != nil {
-		logger.Error(fmt.Sprintf("Failed decoding restart services: %v", err))
+		logger.Errorf("Failed decoding restart services: %v", err)
 		return response.InternalError(err)
 	}
 
 	for _, service := range services {
 		err = ceph.RestartCephService(service.Service)
 		if err != nil {
 			url := s.Address().String()
-			logger.Error(fmt.Sprintf("Failed restarting %s on host %s", service.Service, url))
+			logger.Errorf("Failed restarting %s on host %s", service.Service, url)
 			return response.SyncResponse(false, err)
 		}
 	}

microceph/api/types/configs.go

@@ -5,6 +5,7 @@ package types
 type Config struct {
 	Key   string  `json:"key" yaml:"key"`
 	Value string  `json:"value" yaml:"value"`
+	Wait  bool    `json:"wait" yaml:"wait"`
 }
 
 // Configs is a slice of configs

microceph/ceph/config.go

@@ -14,17 +14,19 @@ import (
 	"github.com/canonical/microceph/microceph/database"
 )
 
+// Config Table is the source of additional information for each supported config key
+// Refer to GetConfigTable()
 type ConfigTable map[string]struct{
 	Who     string // Ceph Config internal <who> against each key
-	Daemons []string // Daemons that need to be restarted post config change.
+	Daemons []string // List of Daemons that need to be restarted across the cluster for the config change to take effect.
 }
 
 // Check if certain key is present in the map.
 func (c ConfigTable) isKeyPresent(key string) bool {
 	if _, ok := c[key]; !ok {
 		return false
 	}
-	
+
 	return true
 }
 
@@ -53,7 +55,7 @@ type ConfigDump []ConfigDumpItem
 
 func SetConfigItem(c types.Config) error {
 	configTable := GetConfigTable()
-	
+
 	args := []string{
 		"config",
 		"set",
@@ -63,34 +65,40 @@ func SetConfigItem(c types.Config) error {
 		"-f",
 		"json-pretty",
 	}
-	
+
 	_, err := processExec.RunCommand("ceph", args...)
 	if err != nil {
 		return err
 	}
-	
+
 	return nil
 }
 
-func GetConfigItem(c types.Config) (types.Config, error) {
+func GetConfigItem(c types.Config) (types.Configs, error) {
 	var err error
 	configTable := GetConfigTable()
-
+	ret := make(types.Configs, 1)
+	who := "mon"
+
+	// workaround to query global configs from mon entity
+	if configTable[c.Key].Who != "global" {
+		who = configTable[c.Key].Who
+	}
+
 	args := []string{
 		"config",
 		"get",
-		configTable[c.Key].Who,
+		who,
 		c.Key,
-		"-f",
-		"json-pretty",
 	}
-
-	c.Value, err = processExec.RunCommand("ceph", args...)
+
+	ret[0].Key = c.Key
+	ret[0].Value, err = processExec.RunCommand("ceph", args...)
 	if err != nil {
-		return c, err
+		return nil, err
 	}
-	
-	return c, nil
+
+	return ret, nil
 }
 
 func RemoveConfigItem(c types.Config) error {
@@ -100,15 +108,13 @@ func RemoveConfigItem(c types.Config) error {
 		"rm",
 		configTable[c.Key].Who,
 		c.Key,
-		"-f",
-		"json-pretty",
 	}
-	
+
 	_, err := processExec.RunCommand("ceph", args...)
 	if err != nil {
 		return err
 	}
-	
+
 	return nil
 }
 
@@ -122,12 +128,12 @@ func ListConfigs() (types.Configs, error) {
 		"-f",
 		"json-pretty",
 	}
-	
+
 	output, err := processExec.RunCommand("ceph", args...)
 	if err != nil {
 		return configs, err
 	}
-	
+
 	json.Unmarshal([]byte(output), &dump)
 	// Only take configs permitted in config table.
 	for _, configItem := range dump {
@@ -138,54 +144,54 @@ func ListConfigs() (types.Configs, error) {
 			})
 		}
 	}
-	
+
 	return configs, nil
 }
 
 // updates the ceph config file.
 func updateConfig(s common.StateInterface) error {
 	confPath := filepath.Join(os.Getenv("SNAP_DATA"), "conf")
 	runPath := filepath.Join(os.Getenv("SNAP_DATA"), "run")
-	
+
 	// Get the configuration and servers.
 	var err error
 	var configItems []database.ConfigItem
 	var monitors []database.Service
-	
+
 	err = s.ClusterState().Database.Transaction(s.ClusterState().Context, func(ctx context.Context, tx *sql.Tx) error {
 		configItems, err = database.GetConfigItems(ctx, tx)
 		if err != nil {
 			return err
 		}
-		
+
 		serviceName := "mon"
 		monitors, err = database.GetServices(ctx, tx, database.ServiceFilter{Service: &serviceName})
 		if err != nil {
 			return err
 		}
-		
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
-	
+
 	config := map[string]string{}
 	for _, item := range configItems {
 		config[item.Key] = item.Value
 	}
-	
+
 	monitorAddresses := make([]string, len(monitors))
 	remotes := s.ClusterState().Remotes().RemotesByName()
 	for i, monitor := range monitors {
 		remote, ok := remotes[monitor.Member]
 		if !ok {
 			continue
 		}
-		
+
 		monitorAddresses[i] = remote.Address.Addr().String()
 	}
-	
+
 	conf := newCephConfig(confPath)
 	address := s.ClusterState().Address().Hostname()
 	err = conf.WriteConfig(
@@ -201,7 +207,7 @@ func updateConfig(s common.StateInterface) error {
 	if err != nil {
 		return fmt.Errorf("Couldn't render ceph.conf: %w", err)
 	}
-	
+
 	// Generate ceph.client.admin.keyring
 	keyring := newCephKeyring(confPath, "ceph.keyring")
 	err = keyring.WriteConfig(
@@ -213,6 +219,6 @@ func updateConfig(s common.StateInterface) error {
 	if err != nil {
 		return fmt.Errorf("Couldn't render ceph.client.admin.keyring: %w", err)
 	}
-	
+
 	return nil
 }