diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index beb4dfaf..54448a9c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,6 +130,36 @@ jobs: s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl put -P ~/test.txt s3://testbucket curl -s http://localhost/testbucket/test.txt | grep -F hello-radosgw + - name: Test Cluster Config + run: | + set -eux + cip=$(ip -4 -j route | jq -r '.[] | select(.dst | contains("default")) | .prefsrc' | tr -d '[:space:]') + + # pre config set timestamp for service age + ts=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2) + + # set config + sudo microceph cluster config set cluster_network $cip/8 --wait + + # post config set timestamp for service age + ts2=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2) + + # Check config output + output=$(sudo microceph cluster config get cluster_network | grep -cim1 'cluster_network') + if [[ $output -lt 1 ]] ; then echo "config check failed: $output"; exit 1; fi + + # Check service restarted + if [ $ts2 -lt $ts ]; then echo "config check failed: TS1: $ts2 TS2: $ts3"; exit 1; fi + + # reset config + sudo microceph cluster config reset cluster_network --wait + + # post config reset timestamp for service age + ts3=$(sudo systemctl show --property ActiveEnterTimestampMonotonic snap.microceph.osd.service | cut -d= -f2) + + # Check service restarted + if [ $ts3 -lt $ts2 ]; then echo "config check failed: TS2: $ts2 TS3: $ts3"; exit 1; fi + - name: Upload artifacts if: always() uses: actions/upload-artifact@v3 diff --git a/microceph/api/configs.go b/microceph/api/configs.go new file mode 100644 index 00000000..c46b1144 --- /dev/null +++ b/microceph/api/configs.go @@ -0,0 +1,89 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/canonical/microcluster/rest" + "github.com/canonical/microcluster/state" + "github.com/lxc/lxd/lxd/response" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" +) + +// /1.0/configs endpoint. +var configsCmd = rest.Endpoint{ + Path: "configs", + + Get: rest.EndpointAction{Handler: cmdConfigsGet, ProxyTarget: true}, + Put: rest.EndpointAction{Handler: cmdConfigsPut, ProxyTarget: true}, + Delete: rest.EndpointAction{Handler: cmdConfigsDelete, ProxyTarget: true}, +} + +func cmdConfigsGet(s *state.State, r *http.Request) response.Response { + var err error + var req types.Config + var configs types.Configs + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // If a valid key string is passed, fetch that key. + if len(req.Key) > 0 { + configs, err = ceph.GetConfigItem(req) + } else { + // Fetch all configs. + configs, err = ceph.ListConfigs() + } + if err != nil { + return response.SmartError(err) + } + + return response.SyncResponse(true, configs) +} + +func cmdConfigsPut(s *state.State, r *http.Request) response.Response { + var req types.Config + configTable := ceph.GetConstConfigTable() + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // Configure the key/value + err = ceph.SetConfigItem(req) + if err != nil { + return response.SmartError(err) + } + + services := configTable[req.Key].Daemons + client.ConfigChangeRefresh(s, services, req.Wait) + + return response.EmptySyncResponse +} + +func cmdConfigsDelete(s *state.State, r *http.Request) response.Response { + var req types.Config + configTable := ceph.GetConstConfigTable() + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // Clean the key/value + err = ceph.RemoveConfigItem(req) + if err != nil { + return response.SmartError(err) + } + + services := configTable[req.Key].Daemons + client.ConfigChangeRefresh(s, services, req.Wait) + + return response.EmptySyncResponse +} diff --git a/microceph/api/endpoints.go b/microceph/api/endpoints.go index 1d7c127d..2fcbba8d 100644 --- a/microceph/api/endpoints.go +++ b/microceph/api/endpoints.go @@ -11,4 +11,6 @@ var Endpoints = []rest.Endpoint{ resourcesCmd, servicesCmd, rgwServiceCmd, + configsCmd, + restartServiceCmd, } diff --git a/microceph/api/services.go b/microceph/api/services.go index da347355..c98f67f6 100644 --- a/microceph/api/services.go +++ b/microceph/api/services.go @@ -2,13 +2,16 @@ package api import ( "encoding/json" + "fmt" + "net/http" + "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/common" - "net/http" "github.com/canonical/microcluster/rest" "github.com/canonical/microcluster/state" "github.com/lxc/lxd/lxd/response" + "github.com/lxc/lxd/shared/logger" "github.com/canonical/microceph/microceph/ceph" ) @@ -29,6 +32,43 @@ func cmdServicesGet(s *state.State, r *http.Request) response.Response { return response.SyncResponse(true, services) } +// Service Reload Endpoint. +var restartServiceCmd = rest.Endpoint{ + Path: "services/restart", + Post: rest.EndpointAction{Handler: cmdRestartServicePost, ProxyTarget: true}, +} + +func cmdRestartServicePost(s *state.State, r *http.Request) response.Response { + var services types.Services + + err := json.NewDecoder(r.Body).Decode(&services) + if err != nil { + logger.Errorf("Failed decoding restart services: %v", err) + return response.InternalError(err) + } + + // Check if provided services are valid and available in microceph + for _, service := range(services) { + valid_services := ceph.GetConfigTableServiceSet() + if _, ok := valid_services[service.Service]; !ok { + err := fmt.Errorf("%s is not a valid ceph service", service.Service) + logger.Errorf("%v", err) + return response.InternalError(err) + } + } + + for _, service := range services { + err = ceph.RestartCephService(service.Service) + if err != nil { + url := s.Address().String() + logger.Errorf("Failed restarting %s on host %s", service.Service, url) + return response.SyncResponse(false, err) + } + } + + return response.EmptySyncResponse +} + var rgwServiceCmd = rest.Endpoint{ Path: "services/rgw", diff --git a/microceph/api/types/configs.go b/microceph/api/types/configs.go new file mode 100644 index 00000000..c3a40b85 --- /dev/null +++ b/microceph/api/types/configs.go @@ -0,0 +1,12 @@ +// Package types provides shared types and structs. +package types + +// Configs holds the key value pair +type Config struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Wait bool `json:"wait" yaml:"wait"` +} + +// Configs is a slice of configs +type Configs []Config diff --git a/microceph/ceph/config.go b/microceph/ceph/config.go index d0f8fd0c..0303b940 100644 --- a/microceph/ceph/config.go +++ b/microceph/ceph/config.go @@ -3,15 +3,164 @@ package ceph import ( "context" "database/sql" + "encoding/json" "fmt" "os" "path/filepath" "strings" + "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/common" "github.com/canonical/microceph/microceph/database" ) +// Config Table is the source of additional information for each supported config key +// Refer to GetConfigTable() +type ConfigTable map[string]struct{ + Who string // Ceph Config internal against each key + Daemons []string // List of Daemons that need to be restarted across the cluster for the config change to take effect. +} + +// Check if certain key is present in the map. +func (c ConfigTable) isKeyPresent(key string) bool { + if _, ok := c[key]; !ok { + return false + } + + return true +} + +// Return keys of the given set +func (c ConfigTable) Keys() (keys []string) { + for k := range c { + keys = append(keys, k) + } + return keys +} + +// Since we can't have const maps, we encapsulate the map into a func +// so that each request for the map gaurantees consistent definition. +func GetConstConfigTable() ConfigTable { + return ConfigTable{ + "public_network": {"global", []string{"mon", "osd"}}, + "cluster_network": {"global", []string{"osd"}}, + } +} + +func GetConfigTableServiceSet() Set { + return Set{ + "mon": struct{}{}, + "mgr": struct{}{}, + "osd": struct{}{}, + "mds": struct{}{}, + "rgw": struct{}{}, + } +} + +// Struct to get Config Items from config dump json output. +type ConfigDumpItem struct{ + Section string + Name string + Value string +} +type ConfigDump []ConfigDumpItem + +func SetConfigItem(c types.Config) error { + configTable := GetConstConfigTable() + + args := []string{ + "config", + "set", + configTable[c.Key].Who, + c.Key, + c.Value, + "-f", + "json-pretty", + } + + _, err := processExec.RunCommand("ceph", args...) + if err != nil { + return err + } + + return nil +} + +func GetConfigItem(c types.Config) (types.Configs, error) { + var err error + configTable := GetConstConfigTable() + ret := make(types.Configs, 1) + who := "mon" + + // workaround to query global configs from mon entity + if configTable[c.Key].Who != "global" { + who = configTable[c.Key].Who + } + + args := []string{ + "config", + "get", + who, + c.Key, + } + + ret[0].Key = c.Key + ret[0].Value, err = processExec.RunCommand("ceph", args...) + if err != nil { + return nil, err + } + + return ret, nil +} + +func RemoveConfigItem(c types.Config) error { + configTable := GetConstConfigTable() + args := []string{ + "config", + "rm", + configTable[c.Key].Who, + c.Key, + } + + _, err := processExec.RunCommand("ceph", args...) + if err != nil { + return err + } + + return nil +} + +func ListConfigs() (types.Configs, error) { + var dump ConfigDump + var configs types.Configs + configTable := GetConstConfigTable() + args := []string{ + "config", + "dump", + "-f", + "json-pretty", + } + + output, err := processExec.RunCommand("ceph", args...) + if err != nil { + return configs, err + } + + json.Unmarshal([]byte(output), &dump) + // Only take configs permitted in config table. + for _, configItem := range dump { + if configTable.isKeyPresent(configItem.Name) { + configs = append(configs, types.Config{ + Key: configItem.Name, + Value: configItem.Value, + }) + } + } + + return configs, nil +} + +// updates the ceph config file. func updateConfig(s common.StateInterface) error { confPath := filepath.Join(os.Getenv("SNAP_DATA"), "conf") runPath := filepath.Join(os.Getenv("SNAP_DATA"), "run") diff --git a/microceph/ceph/config_test.go b/microceph/ceph/config_test.go new file mode 100644 index 00000000..b46dfa15 --- /dev/null +++ b/microceph/ceph/config_test.go @@ -0,0 +1,91 @@ +package ceph + +import ( + "encoding/json" + "testing" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type configSuite struct { + baseSuite +} + +func TestConfig(t *testing.T) { + suite.Run(t, new(configSuite)) +} + +// Set up test suite +func (s *configSuite) SetupTest() { + s.baseSuite.SetupTest() +} + +func addConfigSetExpectations(r *mocks.Runner, key string, value string) { + r.On("RunCommand", []interface{}{ + "ceph", "config", "set", "global", key, value, "-f", "json-pretty", + }...).Return(value, nil).Once() +} + +func addConfigOpExpectations(r *mocks.Runner, op string, key string, value string) { + r.On("RunCommand", []interface{}{ + "ceph", "config", op, "global", key, + }...).Return(value, nil).Once() +} + +func addListConfigExpectations(r *mocks.Runner, key string, value string) { + var configs = ConfigDump{} + configs = append(configs, ConfigDumpItem{Section: "", Name: key, Value: value}) + ret, _ := json.Marshal(configs) + r.On("RunCommand", []interface{}{ + "ceph", "config", "dump", "-f", "json-pretty", + }...).Return(string(ret[:]), nil).Once() +} + +func (s *configSuite) TestSetConfig() { + t := types.Config{Key: "cluster_network", Value: "0.0.0.0/16"} + + r := mocks.NewRunner(s.T()) + addConfigSetExpectations(r, t.Key, t.Value) + processExec = r + + err := SetConfigItem(t) + assert.NoError(s.T(), err) +} + +func (s *configSuite) TestGetConfig() { + t := types.Config{Key: "cluster_network", Value: "0.0.0.0/16"} + + r := mocks.NewRunner(s.T()) + addConfigOpExpectations(r, "get", t.Key, t.Value) + processExec = r + + _, err := GetConfigItem(t) + assert.NoError(s.T(), err) +} + +func (s *configSuite) TestResetConfig() { + t := types.Config{Key: "cluster_network", Value: "0.0.0.0/16"} + + r := mocks.NewRunner(s.T()) + addConfigOpExpectations(r, "rm", t.Key, t.Value) + processExec = r + + err := RemoveConfigItem(t) + assert.NoError(s.T(), err) +} + +func (s *configSuite) TestListConfig() { + t := types.Config{Key: "cluster_network", Value: "0.0.0.0/16"} + + r := mocks.NewRunner(s.T()) + addListConfigExpectations(r, t.Key, t.Value) + processExec = r + + configs, err := ListConfigs() + assert.NoError(s.T(), err) + assert.Equal(s.T(), configs[0].Key, t.Key) + assert.Equal(s.T(), configs[0].Value, t.Value) +} diff --git a/microceph/ceph/osd.go b/microceph/ceph/osd.go index 507ada62..327c3462 100644 --- a/microceph/ceph/osd.go +++ b/microceph/ceph/osd.go @@ -366,7 +366,7 @@ func AddOSD(s *state.State, path string, wipe bool, encrypt bool) error { } // Spawn the OSD. - err = snapReload("osd") + err = snapRestart("osd", true) if err != nil { return err } diff --git a/microceph/ceph/services.go b/microceph/ceph/services.go index ed30b45f..2fb72701 100644 --- a/microceph/ceph/services.go +++ b/microceph/ceph/services.go @@ -4,13 +4,132 @@ import ( "context" "database/sql" "fmt" + "time" + "github.com/Rican7/retry" + "github.com/Rican7/retry/backoff" + "github.com/Rican7/retry/strategy" "github.com/canonical/microcluster/state" + "github.com/lxc/lxd/shared/logger" "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/database" + "github.com/tidwall/gjson" ) +type Set map[string]struct{} + +func (sub Set) isIn(super Set) bool { + flag := true + + // mark flag false if any key from subset is not present in superset. + for key := range sub { + _, ok := super[key] + if !ok { + flag = false + break // Break the loop. + } + } + + return flag +} + +// Table to map fetchFunc for workers (daemons) to a service. +var serviceWorkerTable = map[string](func () (Set, error)) { + "osd": getUpOsds, + "mon": getMons, +} + +// Restarts (in order) all Ceph Services provided in the input slice on the host. +func RestartCephServices(services []string) error { + for i := range services { + err := RestartCephService(services[i]) + if err != nil { + logger.Errorf("Service %s restart failed: %v ", services[i], err) + return err + } + } + + return nil +} + +// Restart provided ceph service ("mon"/"osd"...) on the host. +func RestartCephService(service string) error { + if _, ok := serviceWorkerTable[service]; !ok { + err := fmt.Errorf("No handler defined for service %s", service) + logger.Errorf("%v", err) + return err + } + + // Fetch a Set{} of available daemons for the service. + workers, err := serviceWorkerTable[service]() + if err != nil { + logger.Errorf("Failed fetching service %s workers", service) + return err + } + + // Restart the service. + snapRestart(service, false) + + // Check all the daemons available before Restart are up. + err = retry.Retry(func(i uint) error { + iWorkers, err := serviceWorkerTable[service]() + if err != nil { + return err + } + + // All still not up + if !workers.isIn(iWorkers) { + err := fmt.Errorf( + "Attempt %d: Workers: %v not all present in %v", i, workers, iWorkers, + ) + logger.Errorf("%v", err) + return (err) + } + return nil + }, strategy.Delay(5), strategy.Limit(10), strategy.Backoff(backoff.Linear(10*time.Second))) + if err != nil { + return err + } + + return nil +} + +func getMons() (Set, error) { + retval := Set{} + output, err := processExec.RunCommand("ceph", "mon", "dump", "-f", "json-pretty") + if err != nil { + logger.Errorf("Failed fetching Mon dump: %v", err) + return nil, err + } + + logger.Debugf("Mon Dump:\n%s", output) + // Get a list of mons. + mons := gjson.Get(output, "mons.#.name") + for _, key := range mons.Array() { + retval[key.String()] = struct{}{} + } + + return retval, nil +} + +func getUpOsds() (Set, error) { + retval := Set{} + output, err := processExec.RunCommand("ceph", "osd", "dump", "-f", "json-pretty") + if err != nil { + logger.Errorf("Failed fetching OSD dump: %v", err) + return nil, err + } + + logger.Debugf("OSD Dump:\n%s", output) + // Get a list of uuid of osds in up state. + upOsds := gjson.Get(output, "osds.#(up==1)#.uuid") + for _, element := range upOsds.Array() { + retval[element.String()] = struct{}{} + } + return retval, nil +} + // ListServices retrieves a list of services from the database func ListServices(s *state.State) (types.Services, error) { services := types.Services{} diff --git a/microceph/ceph/services_test.go b/microceph/ceph/services_test.go new file mode 100644 index 00000000..48a276c3 --- /dev/null +++ b/microceph/ceph/services_test.go @@ -0,0 +1,73 @@ +package ceph + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/canonical/microceph/microceph/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type servicesSuite struct { + baseSuite + // TestStateInterface *mocks.StateInterface +} + +func TestServices(t *testing.T) { + suite.Run(t, new(servicesSuite)) +} + +// Set up test suite +func (s *servicesSuite) SetupTest() { + s.baseSuite.SetupTest() +} + +func addOsdDumpExpectations(r *mocks.Runner) { + osdDumpObj := "{\"osds\":[{\"up\":1,\"uuid\":\"bfbbd27a-472f-4771-a6f7-7c5db9803d41\"}]}" + osdDump, _ := json.Marshal(osdDumpObj) + + // Expect osd service worker query + r.On("RunCommand", []interface{}{ + "ceph", "osd", "dump", "-f", "json-pretty", + }...).Return(string(osdDump[:]), nil).Twice() +} + +func addMonDumpExpectations(r *mocks.Runner) { + monDumpObj := "{\"mons\":[{\"name\":\"bfbbd27a\"}]}" + monDump, _ := json.Marshal(monDumpObj) + + // Expect mon service worker query + r.On("RunCommand", []interface{}{ + "ceph", "mon", "dump", "-f", "json-pretty", + }...).Return(string(monDump[:]), nil).Twice() +} + +func addServiceRestartExpectations(r *mocks.Runner, services []string) { + for _, service := range services { + r.On("RunCommand", []interface{}{ + "snapctl", "restart", fmt.Sprintf("microceph.%s", service), + }...).Return("ok", nil).Once() + } +} + +func (s *servicesSuite) TestRestartInvalidService() { + err := RestartCephService("InvalidService") + assert.ErrorContains(s.T(), err, "No handler defined") +} + +func (s *servicesSuite) TestRestartServiceWorkerSuccess() { + ts := []string{"mon", "osd"} // test services + + r := mocks.NewRunner(s.T()) + addMonDumpExpectations(r) + addOsdDumpExpectations(r) + addServiceRestartExpectations(r, ts) + processExec = r + + // Handler is defined for both mon and osd services. + err := RestartCephServices(ts) + assert.Equal(s.T(), err, nil) +} + diff --git a/microceph/ceph/snap.go b/microceph/ceph/snap.go index 4ae6d10b..522c5116 100644 --- a/microceph/ceph/snap.go +++ b/microceph/ceph/snap.go @@ -42,14 +42,18 @@ func snapStop(service string, disable bool) error { return nil } -// snapReload restarts a service via snapctl. -func snapReload(service string) error { +// Restarts (optionally reloads) a service via snapctl. +func snapRestart(service string, isReload bool) error { args := []string{ "restart", - "--reload", - fmt.Sprintf("microceph.%s", service), } + if isReload { + args = append(args, "--reload") + } + + args = append(args, fmt.Sprintf("microceph.%s", service)) + _, err := processExec.RunCommand("snapctl", args...) if err != nil { return err diff --git a/microceph/client/client.go b/microceph/client/client.go index 8b355e81..9f60fa57 100644 --- a/microceph/client/client.go +++ b/microceph/client/client.go @@ -7,11 +7,117 @@ import ( "time" "github.com/canonical/microcluster/client" + "github.com/canonical/microcluster/state" "github.com/lxc/lxd/shared/api" + "github.com/lxc/lxd/shared/logger" "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" ) +func SetConfig(ctx context.Context, c *client.Client, data *types.Config) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "PUT", api.NewURL().Path("configs"), data, nil) + if err != nil { + return fmt.Errorf("Failed setting cluster config: %w, Key: %s, Value: %s", err, data.Key, data.Value) + } + + return nil +} + +func ClearConfig(ctx context.Context, c *client.Client, data *types.Config) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "DELETE", api.NewURL().Path("configs"), data, nil) + if err != nil { + return fmt.Errorf("Failed clearing cluster config: %w, Key: %s", err, data.Key) + } + + return nil +} + +func GetConfig(ctx context.Context, c *client.Client, data *types.Config) (types.Configs, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + configs := types.Configs{} + + err := c.Query(queryCtx, "GET", api.NewURL().Path("configs"), data, &configs) + if err != nil { + return nil, fmt.Errorf("Failed to fetch cluster config: %w, Key: %s", err, data.Key) + } + + return configs, nil +} + +// Perform ordered (one after other) restart of provided Ceph services across the ceph cluster. +func ConfigChangeRefresh(s *state.State, services []string, wait bool) error { + if wait { + // Execute restart synchronously + err := SendRestartRequestToClusterMembers(s, services) + if err != nil { + return err + } + + // Restart on current host. + err = ceph.RestartCephServices(services) + if err != nil { + return err + } + } else { // Execute restart asynchronously + go func() { + SendRestartRequestToClusterMembers(s, services) + ceph.RestartCephServices(services) // Restart on current host. + }() + } + + return nil +} + +func RestartService(ctx context.Context, c *client.Client, data *types.Services) (error) { + // 120 second timeout for waiting. + queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) + defer cancel() + + err := c.Query(queryCtx, "POST", api.NewURL().Path("services", "restart"), data, nil) + if err != nil { + url := c.URL() + return fmt.Errorf("Failed Forwarding To: %s: %w", url.String(), err) + } + + return nil +} + +// Sends the desired list of services to be restarted on every other member of the cluster. +func SendRestartRequestToClusterMembers(s *state.State, services []string) (error) { + // Populate the restart request data. + var data types.Services + for _, service := range services { + data = append(data, types.Service{Service: service}) + } + + // Get a collection of clients to every other cluster member, with the notification user-agent set. + cluster, err := s.Cluster(nil); + if err != nil { + logger.Errorf("Failed to get a client for every cluster member: %v", err) + return err + } + + for _, remoteClient := range cluster { + // In order send restart to each cluster member and wait. + err = RestartService(s.Context, &remoteClient, &data) + if err != nil { + logger.Errorf("Restart error: %v", err) + return err + } + } + + return nil +} + // AddDisk requests Ceph sets up a new OSD. func AddDisk(ctx context.Context, c *client.Client, data *types.DisksPost) error { queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) diff --git a/microceph/cmd/microceph/cluster.go b/microceph/cmd/microceph/cluster.go index f6fa5da4..d8b783bd 100644 --- a/microceph/cmd/microceph/cluster.go +++ b/microceph/cmd/microceph/cluster.go @@ -38,6 +38,10 @@ func (c *cmdCluster) Command() *cobra.Command { clusterSQLCmd := cmdClusterSQL{common: c.common, cluster: c} cmd.AddCommand(clusterSQLCmd.Command()) + // Config Subcommand + clusterConfigCmd := cmdClusterConfig{common: c.common, cluster: c} + cmd.AddCommand(clusterConfigCmd.Command()) + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } diff --git a/microceph/cmd/microceph/cluster_config.go b/microceph/cmd/microceph/cluster_config.go new file mode 100644 index 00000000..3e825882 --- /dev/null +++ b/microceph/cmd/microceph/cluster_config.go @@ -0,0 +1,39 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +type cmdClusterConfig struct { + common *CmdControl + cluster *cmdCluster +} + +func (c *cmdClusterConfig) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Manage Ceph Cluster configs", + } + + // Get + clusterConfigGetCmd := cmdClusterConfigGet{common: c.common, cluster: c.cluster, clusterConfig: c} + cmd.AddCommand(clusterConfigGetCmd.Command()) + + // Set + clusterConfigSetCmd := cmdClusterConfigSet{common: c.common, cluster: c.cluster, clusterConfig: c} + cmd.AddCommand(clusterConfigSetCmd.Command()) + + // Reset + clusterConfigResetCmd := cmdClusterConfigReset{common: c.common, cluster: c.cluster, clusterConfig: c} + cmd.AddCommand(clusterConfigResetCmd.Command()) + + // List + clusterConfigListCmd := cmdClusterConfigList{common: c.common, cluster: c.cluster, clusterConfig: c} + cmd.AddCommand(clusterConfigListCmd.Command()) + + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 + cmd.Args = cobra.NoArgs + cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } + + return cmd +} diff --git a/microceph/cmd/microceph/cluster_config_get.go b/microceph/cmd/microceph/cluster_config_get.go new file mode 100644 index 00000000..1c3ef9fe --- /dev/null +++ b/microceph/cmd/microceph/cluster_config_get.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/lxc/lxd/lxc/utils" + "github.com/spf13/cobra" +) + +type cmdClusterConfigGet struct { + common *CmdControl + cluster *cmdCluster + clusterConfig *cmdClusterConfig +} + +func (c *cmdClusterConfigGet) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "Get specified Ceph Cluster config", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdClusterConfigGet) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetConstConfigTable() + + // Get can be called with a single key. + if len(args) != 1 { + return cmd.Help() + } + + if _, ok := allowList[args[0]]; !ok { + return fmt.Errorf("Key %s is invalid. \nPermitted Keys: %v", args[0], allowList.Keys()) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.Config{ + Key: args[0], + } + + configs, err := client.GetConfig(context.Background(), cli, req) + if err != nil { + return err + } + + data := make([][]string, len(configs)) + for i, config := range configs { + data[i] = []string{fmt.Sprintf("%d", i), config.Key, config.Value} + } + + header := []string{"#", "Key", "Value"} + err = utils.RenderTable(utils.TableFormatTable, header, data, configs) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/cluster_config_list.go b/microceph/cmd/microceph/cluster_config_list.go new file mode 100644 index 00000000..e85246a7 --- /dev/null +++ b/microceph/cmd/microceph/cluster_config_list.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/lxc/lxd/lxc/utils" + "github.com/spf13/cobra" +) + +type cmdClusterConfigList struct { + common *CmdControl + cluster *cmdCluster + clusterConfig *cmdClusterConfig +} + +func (c *cmdClusterConfigList) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List all set Ceph level configs", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdClusterConfigList) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + // Create an empty Key request. + req := &types.Config{ + Key: "", + } + + configs, err := client.GetConfig(context.Background(), cli, req) + if err != nil { + return err + } + + data := make([][]string, len(configs)) + for i, config := range configs { + data[i] = []string{fmt.Sprintf("%d", i), config.Key, config.Value} + } + + header := []string{"#", "Key", "Value"} + err = utils.RenderTable(utils.TableFormatTable, header, data, configs) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/cluster_config_reset.go b/microceph/cmd/microceph/cluster_config_reset.go new file mode 100644 index 00000000..0c1c5ee7 --- /dev/null +++ b/microceph/cmd/microceph/cluster_config_reset.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClusterConfigReset struct { + common *CmdControl + cluster *cmdCluster + clusterConfig *cmdClusterConfig + + flagWait bool +} + +func (c *cmdClusterConfigReset) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "reset ", + Short: "Clear specified Ceph Cluster config", + RunE: c.Run, + } + + cmd.Flags().BoolVar(&c.flagWait, "wait", false, "Wait for required ceph services to restart post config reset.") + return cmd +} + +func (c *cmdClusterConfigReset) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetConstConfigTable() + if len(args) != 1 { + return cmd.Help() + } + + if _, ok := allowList[args[0]]; !ok { + return fmt.Errorf("Resetting key %s is not allowed", args[0]) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.Config{ + Key: args[0], + Wait: c.flagWait, + } + + err = client.ClearConfig(context.Background(), cli, req) + if err != nil { + return err + } + + return nil +} \ No newline at end of file diff --git a/microceph/cmd/microceph/cluster_config_set.go b/microceph/cmd/microceph/cluster_config_set.go new file mode 100644 index 00000000..4894b527 --- /dev/null +++ b/microceph/cmd/microceph/cluster_config_set.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClusterConfigSet struct { + common *CmdControl + cluster *cmdCluster + clusterConfig *cmdClusterConfig + + flagWait bool +} + +func (c *cmdClusterConfigSet) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "set ", + Short: "Set specified Ceph Cluster config", + RunE: c.Run, + } + + cmd.Flags().BoolVar(&c.flagWait, "wait", false, "Wait for required ceph services to restart post config set.") + return cmd +} + +func (c *cmdClusterConfigSet) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetConstConfigTable() + if len(args) != 2 { + return cmd.Help() + } + + if _, ok := allowList[args[0]]; !ok { + return fmt.Errorf("Configuring key %s is not allowed. \nPermitted Keys: %v", args[0], allowList.Keys()) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.Config{ + Key: args[0], + Value: args[1], + Wait: c.flagWait, + } + + err = client.SetConfig(context.Background(), cli, req) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/go.mod b/microceph/go.mod index 373f19af..54e30042 100644 --- a/microceph/go.mod +++ b/microceph/go.mod @@ -5,14 +5,15 @@ go 1.18 require ( github.com/canonical/microcluster v0.0.0-20230501200316-dd78e864d2f1 github.com/lxc/lxd v0.0.0-20230501200206-976cd2bfee6a + github.com/Rican7/retry v0.3.1 github.com/olekukonko/tablewriter v0.0.5 github.com/pborman/uuid v1.2.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 + github.com/tidwall/gjson v1.14.4 ) require ( - github.com/Rican7/retry v0.3.1 // indirect github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a // indirect github.com/canonical/go-dqlite v1.11.9 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -51,6 +52,8 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/zitadel/oidc/v2 v2.5.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/crypto v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect diff --git a/microceph/go.sum b/microceph/go.sum index e54ad57f..7a0a0cb7 100644 --- a/microceph/go.sum +++ b/microceph/go.sum @@ -325,6 +325,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=