From d0b881b0167bb4851727355b9a449410e436144d Mon Sep 17 00:00:00 2001 From: utkarshbhatthere Date: Tue, 9 May 2023 19:38:15 +0530 Subject: [PATCH] Added S3 user management Signed-off-by: utkarshbhatthere --- .github/workflows/tests.yml | 26 +-- docs/conf.py | 2 - microceph/api/endpoints.go | 2 +- microceph/api/s3.go | 79 ++++++++++ microceph/api/types/s3.go | 9 ++ microceph/ceph/rgw_s3.go | 78 +++++++++ microceph/client/s3.go | 69 ++++++++ microceph/cmd/microceph/main.go | 3 + microceph/cmd/microceph/s3.go | 271 ++++++++++++++++++++++++++++++++ scripts/appS3.py | 92 +++++++++++ 10 files changed, 619 insertions(+), 12 deletions(-) create mode 100644 microceph/api/s3.go create mode 100644 microceph/api/types/s3.go create mode 100644 microceph/ceph/rgw_s3.go create mode 100644 microceph/client/s3.go create mode 100644 microceph/cmd/microceph/s3.go create mode 100644 scripts/appS3.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c4c391d..2e9e4246 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,10 +45,24 @@ jobs: runs-on: ubuntu-22.04 needs: build-microceph steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Download snap uses: actions/download-artifact@v3 with: name: snaps + + - name: Install dependencies + run: | + # Python script dependencies + sudo python -m pip install --upgrade pip + sudo pip install flake8 pep8-naming boto3 + + - name: Lint check + run: | + flake8 . --count --show-source --statistics + - name: Install and setup run: | set -eux @@ -160,15 +174,9 @@ jobs: - name: Exercise RGW run: | set -eux - sudo microceph.ceph status - sudo systemctl status snap.microceph.rgw - sudo microceph.radosgw-admin user create --uid=test --display-name=test - sudo microceph.radosgw-admin key create --uid=test --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey - sudo apt-get -qq install s3cmd - echo hello-radosgw > ~/test.txt - s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl mb s3://testbucket - s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl put -P ~/test.txt s3://testbucket - curl -s http://localhost/testbucket/test.txt | grep -F hello-radosgw + sudo microceph s3 create testUser --json > keys.json + sudo python3 ./scripts/appS3.py http://localhost:80 keys.json --obj-num 2 + sudo microceph s3 delete testUser - name: Test Cluster Config run: | diff --git a/docs/conf.py b/docs/conf.py index 814e6612..6bce3bb7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,8 +19,6 @@ templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinxenv'] - - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/microceph/api/endpoints.go b/microceph/api/endpoints.go index cf83185f..2c98f311 100644 --- a/microceph/api/endpoints.go +++ b/microceph/api/endpoints.go @@ -10,11 +10,11 @@ var Endpoints = []rest.Endpoint{ disksCmd, resourcesCmd, servicesCmd, - rgwServiceCmd, configsCmd, restartServiceCmd, mdsServiceCmd, mgrServiceCmd, monServiceCmd, rgwServiceCmd, + s3Cmd, } diff --git a/microceph/api/s3.go b/microceph/api/s3.go new file mode 100644 index 00000000..081bb886 --- /dev/null +++ b/microceph/api/s3.go @@ -0,0 +1,79 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/canonical/lxd/lxd/response" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microcluster/rest" + "github.com/canonical/microcluster/state" +) + +// /1.0/resources endpoint. +var s3Cmd = rest.Endpoint{ + Path: "s3", + Get: rest.EndpointAction{Handler: cmdS3Get, ProxyTarget: true}, + Put: rest.EndpointAction{Handler: cmdS3Put, ProxyTarget: true}, + Delete: rest.EndpointAction{Handler: cmdS3Delete, ProxyTarget: true}, +} + +func cmdS3Get(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // If a user name is passed. + if len(req.Name) > 0 { + getOutput, err := ceph.GetS3User(req) + if err != nil { + return response.SmartError(err) + } + return response.SyncResponse(true, getOutput) + } else { + listOutput, err := ceph.ListS3Users() + if err != nil { + return response.SmartError(err) + } + return response.SyncResponse(true, listOutput) + } +} + +func cmdS3Put(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + output, err := ceph.CreateS3User(req) + if err != nil { + return response.SmartError(err) + } + + return response.SyncResponse(true, output) +} + +func cmdS3Delete(s *state.State, r *http.Request) response.Response { + var err error + var req types.S3User + + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + err = ceph.DeleteS3User(req.Name) + if err != nil { + return response.SmartError(err) + } + + return response.EmptySyncResponse +} diff --git a/microceph/api/types/s3.go b/microceph/api/types/s3.go new file mode 100644 index 00000000..c61f1945 --- /dev/null +++ b/microceph/api/types/s3.go @@ -0,0 +1,9 @@ +// Package types provides shared types and structs. +package types + +// holds the name, access and secretkey required for exposing an S3 user. +type S3User struct { + Name string `json:"name" yaml:"name"` + Key string `json:"key" yaml:"key"` + Secret string `json:"secret" yaml:"secret"` +} \ No newline at end of file diff --git a/microceph/ceph/rgw_s3.go b/microceph/ceph/rgw_s3.go new file mode 100644 index 00000000..3c549110 --- /dev/null +++ b/microceph/ceph/rgw_s3.go @@ -0,0 +1,78 @@ +package ceph + +import ( + "encoding/json" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" +) + +func CreateS3User(user types.S3User) (string, error) { + args := []string{ + "user", + "create", + fmt.Sprintf("--uid=%s", user.Name), + fmt.Sprintf("--display-name=%s", user.Name), + } + + if len(user.Key) > 0 { + args = append(args, fmt.Sprintf("--access-key=%s", user.Key)) + } + + if len(user.Secret) > 0 { + args = append(args, fmt.Sprintf("--secret=%s", user.Secret)) + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return "", err + } + + return output, nil +} + +func GetS3User(user types.S3User) (string, error) { + args := []string{ + "user", + "info", + fmt.Sprintf("--uid=%s", user.Name), + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return "", err + } + + return output, nil +} + +func ListS3Users() ([]string, error) { + args := []string{ + "user", + "list", + } + + output, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return []string{}, err + } + + ret := []string{} + json.Unmarshal([]byte(output), &ret) + return ret, nil +} + +func DeleteS3User(name string) error { + args := []string{ + "user", + "rm", + fmt.Sprintf("--uid=%s", name), + } + + _, err := processExec.RunCommand("radosgw-admin", args...) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/client/s3.go b/microceph/client/s3.go new file mode 100644 index 00000000..83116fb7 --- /dev/null +++ b/microceph/client/s3.go @@ -0,0 +1,69 @@ +// Package client provides a full Go API client. +package client + +import ( + "context" + "time" + + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microcluster/client" +) + +func GetS3User(ctx context.Context, c *client.Client, user *types.S3User) (string, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := "" + err := c.Query(queryCtx, "GET", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func ListS3Users(ctx context.Context, c *client.Client) ([]string, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := []string{} // List of usernames + // GET request with no user name fetches all users. + err := c.Query(queryCtx, "GET", api.NewURL().Path("s3"), &types.S3User{Name: ""}, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func CreateS3User(ctx context.Context, c *client.Client, user *types.S3User) (string, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := "" + err := c.Query(queryCtx, "PUT", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return ret, err + } + + return ret, nil +} + +func DeleteS3User(ctx context.Context, c *client.Client, user *types.S3User) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + + ret := types.S3User{} + err := c.Query(queryCtx, "DELETE", api.NewURL().Path("s3"), user, &ret) + if err != nil { + logger.Error(err.Error()) + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/main.go b/microceph/cmd/microceph/main.go index 36d99810..a0f412e7 100644 --- a/microceph/cmd/microceph/main.go +++ b/microceph/cmd/microceph/main.go @@ -61,6 +61,9 @@ func main() { var cmdDisk = cmdDisk{common: &commonCmd} app.AddCommand(cmdDisk.Command()) + var cmdS3 = cmdS3{common: &commonCmd} + app.AddCommand(cmdS3.Command()) + app.InitDefaultHelpCmd() err := app.Execute() diff --git a/microceph/cmd/microceph/s3.go b/microceph/cmd/microceph/s3.go new file mode 100644 index 00000000..6b410f15 --- /dev/null +++ b/microceph/cmd/microceph/s3.go @@ -0,0 +1,271 @@ +package main + +import ( + "context" + "fmt" + + lxdCmd "github.com/canonical/lxd/shared/cmd" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" + "github.com/tidwall/gjson" +) + +type cmdS3 struct { + common *CmdControl +} + +type cmdS3Get struct { + common *CmdControl + s3 *cmdS3 + jsonOutput bool +} + +type cmdS3Create struct { + common *CmdControl + s3 *cmdS3 + accessKey string + secret string + jsonOutput bool +} + +type cmdS3Delete struct { + common *CmdControl + s3 *cmdS3 +} + +type cmdS3List struct { + common *CmdControl + s3 *cmdS3 +} + +// parent s3 command handle +func (c *cmdS3) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "s3", + Short: "Manage S3 users for Object storage", + } + + // Create + s3CreateCmd := cmdS3Create{common: c.common, s3: c} + cmd.AddCommand(s3CreateCmd.Command()) + + // Delete + s3DeleteCmd := cmdS3Delete{common: c.common, s3: c} + cmd.AddCommand(s3DeleteCmd.Command()) + + // Get + s3GetCmd := cmdS3Get{common: c.common, s3: c} + cmd.AddCommand(s3GetCmd.Command()) + + // List + s3ListCmd := cmdS3List{common: c.common, s3: c} + cmd.AddCommand(s3ListCmd.Command()) + + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 + cmd.Args = cobra.NoArgs + cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } + + return cmd +} + +// s3 Get command handle +func (c *cmdS3Get) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "Fetch details of an existing S3 user", + RunE: c.Run, + } + + cmd.Flags().BoolVar(&c.jsonOutput, "json", false, "Provide output in json format") + return cmd +} + +func (c *cmdS3Get) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to fetch S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + input := &types.S3User{Name: args[0]} + user, err := client.GetS3User(context.Background(), cli, input) + if err != nil { + return err + } + + err = renderOutput(user, c.jsonOutput) + if err != nil { + return err + } + + return nil +} + +// s3 create command handle +func (c *cmdS3Create) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "create ", + Short: "Create a new S3 user", + RunE: c.Run, + } + + cmd.Flags().StringVar(&c.accessKey, "access-key", "", "custom access-key for new S3 user.") + cmd.Flags().StringVar(&c.secret, "secret", "", "custom secret for new S3 user.") + cmd.Flags().BoolVar(&c.jsonOutput, "json", false, "Provide output in json format") + return cmd +} + +func (c *cmdS3Create) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to create S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + // Create a user with given keys. + input := &types.S3User{ + Name: args[0], + Key: c.accessKey, + Secret: c.secret, + } + user, err := client.CreateS3User(context.Background(), cli, input) + if err != nil { + return err + } + + err = renderOutput(user, c.jsonOutput) + if err != nil { + return err + } + + return nil +} + +// s3 delete command handle +func (c *cmdS3Delete) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an existing S3 user", + RunE: c.Run, + } + return cmd +} + +func (c *cmdS3Delete) Run(cmd *cobra.Command, args []string) error { + // Get should be called with a single name param. + if len(args) != 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to delete S3 user: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + err = client.DeleteS3User(context.Background(), cli, &types.S3User{Name: args[0]}) + if err != nil { + return err + } + + return nil +} + +// s3 list command handle +func (c *cmdS3List) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List all existing S3 users", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdS3List) Run(cmd *cobra.Command, args []string) error { + // Should not be called with any params + if len(args) > 1 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to list S3 users: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + users, err := client.ListS3Users(context.Background(), cli) + if err != nil { + return err + } + + data := make([][]string, len(users)) + for i := range users { + data[i] = []string{fmt.Sprintf("%d", i+1), users[i]} + } + + header := []string{"#", "Name"} + err = lxdCmd.RenderTable(lxdCmd.TableFormatTable, header, data, users) + if err != nil { + return err + } + + return nil +} + +func renderOutput(output string, isJson bool) error { + if isJson { + fmt.Print(output) + } else { + user := types.S3User{ + Name: gjson.Get(output, "keys.0.user").Str, + Key: gjson.Get(output, "keys.0.access_key").Str, + Secret: gjson.Get(output, "keys.0.secret_key").Str, + } + err := renderSingleS3User(user) + if err != nil { + return err + } + } + return nil +} + +func renderSingleS3User(user types.S3User) error { + data := make([][]string, 1) + data[0] = []string{user.Name, user.Key, user.Secret} + + header := []string{"Name", "Access Key", "Secret"} + err := lxdCmd.RenderTable(lxdCmd.TableFormatTable, header, data, user) + if err != nil { + return err + } + return nil +} diff --git a/scripts/appS3.py b/scripts/appS3.py new file mode 100644 index 00000000..3c749e5c --- /dev/null +++ b/scripts/appS3.py @@ -0,0 +1,92 @@ +import string +import random +import boto3 +import json +import argparse + + +def app_handle(args): + keys_path = args.keys + endpoint = args.endpoint + + # Fetch Auth + with open(keys_path, 'r') as keys_file: + keys_dict = json.load(keys_file) + + # Create Boto3 Client + keys = keys_dict["keys"][0] + client = boto3.resource("s3", verify=False, + endpoint_url=endpoint, + aws_access_key_id=keys["access_key"], + aws_secret_access_key=keys["secret_key"]) + + # Perform IO + objects = [] + bucket_name = "test-bucket" + client.Bucket(bucket_name).create() + for i in range(args.obj_num): + object_name = "test-object"+rand_str(4) + data = str(rand_str(random.randint(10, 30)))*1024*1024 + primary_object_one = client.Object( + bucket_name, + object_name + ) + primary_object_one.put(Body=data) + # Store for + objects.append( + (object_name, primary_object_one.content_length/(1024*1024)) + ) + + # Print Summary + print("IO Summary: Object Count {}".format(args.obj_num)) + for obj, size in objects: + print("Object: {}/{} -> Size: {}MB".format(bucket_name, obj, size)) + + # Cleanup (if asked for) + if not args.no_delete: + print("Performing Cleanup") + for obj, size in objects: + client.Object(bucket_name, obj).delete() + client.Bucket(bucket_name).delete() + + +def rand_str(length: int): + return "".join( + random.choices(string.ascii_uppercase + string.digits, k=length) + ) + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser( + description="An application which uses S3 for storage", + epilog="Ex: python3 appS3.py --keys keys.txt", + ) + + argparse.add_argument( + "endpoint", + type=str, + help="Provide RGW endpoint to talk to.", + ) + argparse.add_argument( + "keys", + type=str, + help="Provide JSON file generated from Ceph RGW Admin.", + ) + argparse.add_argument( + "--obj-num", + type=int, + default=1, + help="Number of objects to upload to S3.", + ) + argparse.add_argument( + "--no-delete", + action="store_true", + help="Setting this to true would not cleanup the pushed objects.", + ) + argparse.set_defaults(func=app_handle) + + # Parse the args. + args = argparse.parse_args() + + # Call the subcommand. + args.func(args)