Added S3 user management

Signed-off-by: Utkarsh Bhatt <[email protected]>

.github/workflows/tests.yml

@@ -81,6 +81,16 @@ jobs:
     - name: Free disk
       run: ~/actionutils.sh free_runner_disk
 
+    - name: Install dependencies
+      run: |
+        # Python script dependencies
+        sudo python -m pip install --upgrade pip
+        sudo pip install flake8 pep8-naming boto3
+
+    - name: Lint check
+      run: |
+        flake8 . --count --show-source --statistics
+
     - name: Install and setup
       run: |
         ~/actionutils.sh install_microceph

docs/how-to/s3-user.rst

@@ -0,0 +1,106 @@
+Use S3 user management on MicroCeph
+===================================
+
+MicroCeph provides an easy to use interface for creating, viewing and deleting s3 users for interfacing with the RGW endpoint.
+This enables smooth and easy access to Object Storage.
+
+.. list-table:: Supported s3-user operations
+   :widths: 30 70
+   :header-rows: 1
+
+   * - Operation
+     - Description
+   * - create
+     - Create provided s3 (radosgw) user with optionally provided access-key and secret
+   * - delete
+     - Delete provided s3 (radosgw) user
+   * - get
+     - Fetch key information of the provided s3 (radosgw) user
+   * - list
+     - List all s3 (radosgw) users
+.. note:: Users can additionally provide --json flag to create and get commands to dump a much detailed 
+
+1. Create an S3 user (optionally provide --access-key --secret and --json)
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user create newTestUser --access-key=ThisIsAccessKey --secret=ThisIsSecret --json
+    {
+        "user_id": "newTestUser",
+        "display_name": "newTestUser",
+        "email": "",
+        "suspended": 0,
+        "max_buckets": 1000,
+        "subusers": [],
+        "keys": [
+            {
+                "user": "newTestUser",
+                "access_key": "ThisIsAccessKey",
+                "secret_key": "ThisIsSecret"
+            }
+        ],
+        "swift_keys": [],
+        "caps": [],
+        "op_mask": "read, write, delete",
+        "default_placement": "",
+        "default_storage_class": "",
+        "placement_tags": [],
+        "bucket_quota": {
+            "enabled": false,
+            "check_on_raw": false,
+            "max_size": -1,
+            "max_size_kb": 0,
+            "max_objects": -1
+        },
+        "user_quota": {
+            "enabled": false,
+            "check_on_raw": false,
+            "max_size": -1,
+            "max_size_kb": 0,
+            "max_objects": -1
+        },
+        "temp_url_keys": [],
+        "type": "rgw",
+        "mfa_ids": []
+    }
+
+2. List all s3 users :
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user list
+    +---+-------------+
+    | # |    NAME     |
+    +---+-------------+
+    | 1 | newTestUser |
+    +---+-------------+
+    | 2 | testUser    |
+    +---+-------------+
+
+3. Get details of a an s3 user (optionally use --json flag to get complete details):
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user get testUser
+    +----------+----------------------+---------+
+    |   NAME   |    ACCESS KEY   |    SECRET    |
+    +----------+----------------------+---------+
+    | testUser | ThisIsAccessKey | ThisIsSecret |
+    +----------+----------------------+---------+
+
+4. Delete an s3 user:
+
+  .. code-block:: shell
+
+   $ sudo microceph s3-user delete newTestUser
+   $ sudo microceph s3-user list
+    +---+----------+
+    | # |   NAME   |
+    +---+----------+
+    | 1 | testUser |
+    +---+----------+
+
+  .. warning:: All the related buckets+objects should be deleted before deletion of the user. 
+
+For more fine-tuned user management use `radosgw-admin CLI <https://docs.ceph.com/en/latest/man/8/radosgw-admin/>`_
+

microceph/api/endpoints.go

@@ -11,7 +11,6 @@ var Endpoints = []rest.Endpoint{
 	disksDelCmd,
 	resourcesCmd,
 	servicesCmd,
-	rgwServiceCmd,
 	configsCmd,
 	restartServiceCmd,
 	mdsServiceCmd,
@@ -21,4 +20,5 @@ var Endpoints = []rest.Endpoint{
 	clientCmd,
 	clientConfigsCmd,
 	clientConfigsKeyCmd,
+	s3Cmd,
 }

microceph/api/s3.go

@@ -0,0 +1,79 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/canonical/lxd/lxd/response"
+	"github.com/canonical/microceph/microceph/api/types"
+	"github.com/canonical/microceph/microceph/ceph"
+	"github.com/canonical/microcluster/rest"
+	"github.com/canonical/microcluster/state"
+)
+
+// /1.0/resources endpoint.
+var s3Cmd = rest.Endpoint{
+	Path:   "services/rgw/user",
+	Get:    rest.EndpointAction{Handler: cmdS3Get, ProxyTarget: true},
+	Put:    rest.EndpointAction{Handler: cmdS3Put, ProxyTarget: true},
+	Delete: rest.EndpointAction{Handler: cmdS3Delete, ProxyTarget: true},
+}
+
+func cmdS3Get(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	// If a user name is passed.
+	if len(req.Name) > 0 {
+		getOutput, err := ceph.GetS3User(req)
+		if err != nil {
+			return response.SmartError(err)
+		}
+		return response.SyncResponse(true, getOutput)
+	} else {
+		listOutput, err := ceph.ListS3Users()
+		if err != nil {
+			return response.SmartError(err)
+		}
+		return response.SyncResponse(true, listOutput)
+	}
+}
+
+func cmdS3Put(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	output, err := ceph.CreateS3User(req)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	return response.SyncResponse(true, output)
+}
+
+func cmdS3Delete(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	err = ceph.DeleteS3User(req.Name)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	return response.EmptySyncResponse
+}

microceph/api/types/s3.go

@@ -0,0 +1,9 @@
+// Package types provides shared types and structs.
+package types
+
+// holds the name, access and secretkey required for exposing an S3 user.
+type S3User struct {
+	Name     string `json:"name" yaml:"name"`
+	Key      string `json:"key" yaml:"key"`
+	Secret   string `json:"secret" yaml:"secret"`
+}

microceph/ceph/rgw_s3.go

@@ -0,0 +1,78 @@
+package ceph
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/canonical/microceph/microceph/api/types"
+)
+
+func CreateS3User(user types.S3User) (string, error) {
+	args := []string{
+		"user",
+		"create",
+		fmt.Sprintf("--uid=%s", user.Name),
+		fmt.Sprintf("--display-name=%s", user.Name),
+	}
+
+	if len(user.Key) > 0 {
+		args = append(args, fmt.Sprintf("--access-key=%s", user.Key))
+	}
+
+	if len(user.Secret) > 0 {
+		args = append(args, fmt.Sprintf("--secret=%s", user.Secret))
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return "", err
+	}
+
+	return output, nil
+}
+
+func GetS3User(user types.S3User) (string, error) {
+	args := []string{
+		"user",
+		"info",
+		fmt.Sprintf("--uid=%s", user.Name),
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return "", err
+	}
+
+	return output, nil
+}
+
+func ListS3Users() ([]string, error) {
+	args := []string{
+		"user",
+		"list",
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return []string{}, err
+	}
+
+	ret := []string{}
+	json.Unmarshal([]byte(output), &ret)
+	return ret, nil
+}
+
+func DeleteS3User(name string) error {
+	args := []string{
+		"user",
+		"rm",
+		fmt.Sprintf("--uid=%s", name),
+	}
+
+	_, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

microceph/client/s3.go

@@ -0,0 +1,69 @@
+// Package client provides a full Go API client.
+package client
+
+import (
+	"context"
+	"time"
+
+	"github.com/canonical/lxd/shared/api"
+	"github.com/canonical/lxd/shared/logger"
+	"github.com/canonical/microceph/microceph/api/types"
+	"github.com/canonical/microcluster/client"
+)
+
+func GetS3User(ctx context.Context, c *client.Client, user *types.S3User) (string, error) {
+	queryCtx, cancel := context.WithTimeout(ctx, time.Second*60)
+	defer cancel()
+
+	ret := ""
+	err := c.Query(queryCtx, "GET", api.NewURL().Path("services", "rgw", "user"), user, &ret)
+	if err != nil {
+		logger.Error(err.Error())
+		return ret, err
+	}
+
+	return ret, nil
+}
+
+func ListS3Users(ctx context.Context, c *client.Client) ([]string, error) {
+	queryCtx, cancel := context.WithTimeout(ctx, time.Second*60)
+	defer cancel()
+
+	ret := []string{} // List of usernames
+	// GET request with no user name fetches all users.
+	err := c.Query(queryCtx, "GET", api.NewURL().Path("services", "rgw", "user"), &types.S3User{Name: ""}, &ret)
+	if err != nil {
+		logger.Error(err.Error())
+		return ret, err
+	}
+
+	return ret, nil
+}
+
+func CreateS3User(ctx context.Context, c *client.Client, user *types.S3User) (string, error) {
+	queryCtx, cancel := context.WithTimeout(ctx, time.Second*60)
+	defer cancel()
+
+	ret := ""
+	err := c.Query(queryCtx, "PUT", api.NewURL().Path("services", "rgw", "user"), user, &ret)
+	if err != nil {
+		logger.Error(err.Error())
+		return ret, err
+	}
+
+	return ret, nil
+}
+
+func DeleteS3User(ctx context.Context, c *client.Client, user *types.S3User) error {
+	queryCtx, cancel := context.WithTimeout(ctx, time.Second*60)
+	defer cancel()
+
+	ret := types.S3User{}
+	err := c.Query(queryCtx, "DELETE", api.NewURL().Path("services", "rgw", "user"), user, &ret)
+	if err != nil {
+		logger.Error(err.Error())
+		return err
+	}
+
+	return nil
+}

microceph/cmd/microceph/main.go

@@ -66,6 +66,9 @@ func main() {
 	var cmdClient = cmdClient{common: &commonCmd}
 	app.AddCommand(cmdClient.Command())
 
+	var cmdS3 = cmdS3{common: &commonCmd}
+	app.AddCommand(cmdS3.Command())
+
 	app.InitDefaultHelpCmd()
 
 	err := app.Execute()