diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 54448a9c..403bc3ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -122,8 +122,7 @@ jobs: - name: Exercise RGW run: | set -eux - sudo microceph.radosgw-admin user create --uid=test --display-name=test - sudo microceph.radosgw-admin key create --uid=test --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey + sudo microceph s3 create test --access-key=fooAccessKey --secret=fooSecretKey sudo apt-get -qq install s3cmd echo hello-radosgw > ~/test.txt s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl mb s3://testbucket diff --git a/microceph/api/endpoints.go b/microceph/api/endpoints.go index 2fcbba8d..059daccc 100644 --- a/microceph/api/endpoints.go +++ b/microceph/api/endpoints.go @@ -13,4 +13,5 @@ var Endpoints = []rest.Endpoint{ rgwServiceCmd, configsCmd, restartServiceCmd, + s3Cmd, } diff --git a/microceph/api/s3.go b/microceph/api/s3.go new file mode 100644 index 00000000..fdf8bf11 --- /dev/null +++ b/microceph/api/s3.go @@ -0,0 +1,79 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microcluster/rest" + "github.com/canonical/microcluster/state" + "github.com/lxc/lxd/lxd/response" +) + +// /1.0/resources endpoint. +var s3Cmd = rest.Endpoint{ + Path: "s3", + Get: rest.EndpointAction{Handler: cmdS3Get, ProxyTarget: true}, + Put: rest.EndpointAction{Handler: cmdS3Put, ProxyTarget: true}, + Delete: rest.EndpointAction{Handler: cmdS3Delete, ProxyTarget: true}, +} + +func cmdS3Get(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // If a user name is passed. + if len(req.Name) > 0 { + getOutput, err := ceph.GetS3User(req.Name) + if err != nil { + return response.SmartError(err) + } + return response.SyncResponse(true, getOutput) + } else { + listOutput, err := ceph.ListS3Users() + if err != nil { + return response.SmartError(err) + } + return response.SyncResponse(true, listOutput) + } +} + +func cmdS3Put(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + output, err := ceph.CreateS3User(req.Name, req.Key, req.Secret) + if err != nil { + return response.SmartError(err) + } + + return response.SyncResponse(true, output) +} + +func cmdS3Delete(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + err = ceph.DeleteS3User(req.Name) + if err != nil { + return response.SmartError(err) + } + + return response.EmptySyncResponse +} \ No newline at end of file diff --git a/microceph/api/types/s3.go b/microceph/api/types/s3.go new file mode 100644 index 00000000..c61f1945 --- /dev/null +++ b/microceph/api/types/s3.go @@ -0,0 +1,9 @@ +// Package types provides shared types and structs. +package types + +// holds the name, access and secretkey required for exposing an S3 user. +type S3User struct { + Name string `json:"name" yaml:"name"` + Key string `json:"key" yaml:"key"` + Secret string `json:"secret" yaml:"secret"` +} \ No newline at end of file diff --git a/microceph/ceph/rgw_s3.go b/microceph/ceph/rgw_s3.go new file mode 100644 index 00000000..46b6293a --- /dev/null +++ b/microceph/ceph/rgw_s3.go @@ -0,0 +1,89 @@ +package ceph + +import ( + "encoding/json" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/tidwall/gjson" +) + +func CreateS3User(name string, accessKey string, secret string) (types.S3User, error) { + args := []string{ + "user", + "create", + fmt.Sprintf("--uid=%s", name), + fmt.Sprintf("--display-name=%s", name), + } + + if len(accessKey) > 0 { + args = append(args, fmt.Sprintf("--access-key=%s", accessKey)) + } + + if len(secret) > 0 { + args = append(args, fmt.Sprintf("--secret=%s", secret)) + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return types.S3User{}, err + } + + ret := types.S3User{ + Name: gjson.Get(output, "keys.0.user").Str, + Key: gjson.Get(output, "keys.0.access_key").Str, + Secret: gjson.Get(output, "keys.0.secret_key").Str, + } + return ret, nil +} + +func GetS3User(name string) (types.S3User, error) { + args := []string{ + "user", + "info", + fmt.Sprintf("--uid=%s", name), + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return types.S3User{}, err + } + + ret := types.S3User{ + Name: gjson.Get(output, "keys.0.user").Str, + Key: gjson.Get(output, "keys.0.access_key").Str, + Secret: gjson.Get(output, "keys.0.secret_key").Str, + } + return ret, nil +} + +func ListS3Users() ([]string, error) { + args := []string{ + "user", + "list", + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return []string{}, err + } + + ret := []string{} + json.Unmarshal([]byte(output), &ret) + return ret, nil +} + +func DeleteS3User(name string) error { + args := []string{ + "user", + "rm", + fmt.Sprintf("--uid=%s", name), + } + + _, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/client/s3.go b/microceph/client/s3.go new file mode 100644 index 00000000..a7200fb1 --- /dev/null +++ b/microceph/client/s3.go @@ -0,0 +1,69 @@ +// Package client provides a full Go API client. +package client + +import ( + "context" + "time" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microcluster/client" + "github.com/lxc/lxd/shared/api" + "github.com/lxc/lxd/shared/logger" +) + +func GetS3User(ctx context.Context, c *client.Client, user *types.S3User) (types.S3User, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := types.S3User{} + err := c.Query(queryCtx, "GET", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func ListS3Users(ctx context.Context, c *client.Client) ([]string, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := []string{} // List of usernames + // GET request with no user name fetches all users. + err := c.Query(queryCtx, "GET", api.NewURL().Path("s3"), &types.S3User{Name: ""}, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func CreateS3User(ctx context.Context, c *client.Client, user *types.S3User) (types.S3User, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := types.S3User{} + err := c.Query(queryCtx, "PUT", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func DeleteS3User(ctx context.Context, c *client.Client, user *types.S3User) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := types.S3User{} + err := c.Query(queryCtx, "DELETE", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return err + } + + return nil +} \ No newline at end of file diff --git a/microceph/cmd/microceph/main.go b/microceph/cmd/microceph/main.go index 36d99810..a0f412e7 100644 --- a/microceph/cmd/microceph/main.go +++ b/microceph/cmd/microceph/main.go @@ -61,6 +61,9 @@ func main() { var cmdDisk = cmdDisk{common: &commonCmd} app.AddCommand(cmdDisk.Command()) + var cmdS3 = cmdS3{common: &commonCmd} + app.AddCommand(cmdS3.Command()) + app.InitDefaultHelpCmd() err := app.Execute() diff --git a/microceph/cmd/microceph/s3.go b/microceph/cmd/microceph/s3.go new file mode 100644 index 00000000..264daccf --- /dev/null +++ b/microceph/cmd/microceph/s3.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/lxc/lxd/lxc/utils" + "github.com/spf13/cobra" +) + +type cmdS3 struct { + common *CmdControl +} + +type cmdS3Get struct { + common *CmdControl + s3 *cmdS3 +} + +type cmdS3Create struct { + common *CmdControl + s3 *cmdS3 + accessKey string + secret string +} + +type cmdS3Delete struct { + common *CmdControl + s3 *cmdS3 +} + +type cmdS3List struct { + common *CmdControl + s3 *cmdS3 +} + +// parent s3 command handle +func (c *cmdS3) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "s3", + Short: "Manage S3 users for Object storage", + } + + // Create + s3CreateCmd := cmdS3Create{common: c.common, s3: c} + cmd.AddCommand(s3CreateCmd.Command()) + + // Delete + s3DeleteCmd := cmdS3Delete{common: c.common, s3: c} + cmd.AddCommand(s3DeleteCmd.Command()) + + // Get + s3GetCmd := cmdS3Get{common: c.common, s3: c} + cmd.AddCommand(s3GetCmd.Command()) + + // List + s3ListCmd := cmdS3List{common: c.common, s3: c} + cmd.AddCommand(s3ListCmd.Command()) + + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 + cmd.Args = cobra.NoArgs + cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } + + return cmd +} + +// s3 Get command handle +func (c* cmdS3Get) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "Fetch details of an existing S3 user", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdS3Get) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to fetch S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + user, err := client.GetS3User(context.Background(), cli, &types.S3User{Name: args[0]}) + if err != nil { + return err + } + + err = renderSingleS3User(user) + if err != nil { + return err + } + + return nil +} + +// s3 create command handle +func (c* cmdS3Create) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "create ", + Short: "Create a new S3 user", + RunE: c.Run, + } + + cmd.Flags().StringVar(&c.accessKey, "access-key", "", "custom access-key for new S3 user.") + cmd.Flags().StringVar(&c.secret, "secret", "", "custom secret for new S3 user.") + return cmd +} + +func (c *cmdS3Create) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to create S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + // Create a user with given keys. + input := &types.S3User{ + Name: args[0], + Key: c.accessKey, + Secret: c.secret, + } + user, err := client.CreateS3User(context.Background(), cli, input) + if err != nil { + return err + } + + err = renderSingleS3User(user) + if err != nil { + return err + } + + return nil +} + +// s3 delete command handle +func (c* cmdS3Delete) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an existing S3 user", + RunE: c.Run, + } + return cmd +} + +func (c *cmdS3Delete) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to delete S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + err = client.DeleteS3User(context.Background(), cli, &types.S3User{Name: args[0]}) + if err != nil { + return err + } + + return nil +} + +// s3 list command handle +func (c* cmdS3List) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List all existing S3 users", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdS3List) Run(cmd *cobra.Command, args []string) error { + // Should not be called with any params + if len(args) > 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("Unable to list S3 users: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + users, err := client.ListS3Users(context.Background(), cli) + if err != nil { + return err + } + + data := make([][]string, len(users)) + for i := range(users) { + data[i] = []string{fmt.Sprintf("%d", i+1), users[i]} + } + + header := []string{"#", "Name"} + err = utils.RenderTable(utils.TableFormatTable, header, data, users) + if err != nil { + return err + } + + return nil +} + +func renderSingleS3User(user types.S3User) error { + data := make([][]string, 1) + data[0] = []string{user.Name, user.Key, user.Secret} + + header := []string{"Name", "Access Key", "Secret"} + err := utils.RenderTable(utils.TableFormatTable, header, data, user) + if err != nil { + return err + } + return nil +}