Add OSD encryption

Add optional OSD disk encryption via LUKS and dm-crypt

Signed-off-by: Peter Sabaini <[email protected]>

.github/workflows/tests.yml

@@ -38,6 +38,10 @@ jobs:
           sudo snap install --dangerous microceph_*.snap
           sudo snap connect microceph:block-devices
           sudo snap connect microceph:hardware-observe
+          sudo snap connect microceph:dm-crypt
+
+          # Daemon needs restart for the added dm-crypt connection
+          sudo snap restart microceph.daemon
 
           sudo microceph cluster bootstrap
           sudo microceph.ceph version
@@ -68,7 +72,7 @@ jobs:
             # names (/dev/sdiY) that are not used inside GitHub Action runners
             minor="${loop_dev##/dev/loop}"
             sudo mknod -m 0660 "/dev/sdi${l}" b 7 "${minor}"
-            sudo microceph disk add --wipe "/dev/sdi${l}"
+            sudo microceph disk add --wipe "/dev/sdi${l}" --encrypt
           done
 
           # Wait for OSDs to become up
@@ -133,3 +137,4 @@ jobs:
           name: snaps
           path: "*.snap"
           retention-days: 5
+

docs/explanation/fde-osd.rst

@@ -0,0 +1,38 @@
+Full Disk Encryption on OSDs
+============================
+
+
+Overview
+--------
+
+MicroCeph supports automatic full disk encryption (FDE) on OSDs.
+
+Full disk encryption is a security measure that protects the data on a storage device by encrypting all the information on the disk. FDE helps maintain data confidentiality in case the disk is lost or stolen by rendering the data inaccessible without the correct decryption key or password.
+
+In the event of disk loss or theft, unauthorized individuals are unable to access the encrypted data, as the encryption renders the information unreadable without the proper credentials. This helps prevent data breaches and protects sensitive information from being misused.
+
+FDE also eliminates the need for wiping or physically destroying a disk when it is replaced, as the encrypted data remains secure even if the disk is no longer in use. The data on the disk is effectively rendered useless without the decryption key.
+
+
+Implementation
+--------------
+
+Full disk encryption for OSDs has to be requested when adding disks. MicroCeph will then generate a random key, store it in the Ceph cluster configuration, and use it to encrypt the given disk via `LUKS/cryptsetup <https://gitlab.com/cryptsetup/cryptsetup/-/wikis/home>`_.
+
+
+Limitations
+-----------
+
+**Warning:** It is important to note that MicroCeph FDE *only* encompasses OSDs. Other data, such as state information for monitors, logs, configuration etc., will *not* be encrypted by this mechanism.
+
+
+Usage
+-----
+
+FDE for OSDs is activated by passing the optional ``--encrypt`` flag when adding disks:
+
+.. code-block:: shell
+
+    sudo microceph disk add /dev/sdx --wipe --encrypt
+
+Note there is no facility to encrypt an OSD that is already part of the cluster. To enable encryption you will have to take the OSD disk out of the cluster, ensure data is replicated and the cluster converged and is healthy, and then re-introduce the OSD with encryption.

docs/explanation/index.rst

@@ -1,4 +1,10 @@
 Explanation
 =============
 
-Coming soon...
+Discussion and clarification of key topics
+
+.. toctree::
+   :maxdepth: 1
+
+   fde-osd
+

microceph/api/disks.go

@@ -37,7 +37,7 @@ func cmdDisksPost(s *state.State, r *http.Request) response.Response {
 		return response.InternalError(err)
 	}
 
-	err = ceph.AddOSD(s, req.Path, req.Wipe)
+	err = ceph.AddOSD(s, req.Path, req.Wipe, req.Encrypt)
 	if err != nil {
 		return response.SmartError(err)
 	}

microceph/api/types/disks.go

@@ -3,8 +3,9 @@ package types
 
 // DisksPost hold a path and a flag for enabling device wiping
 type DisksPost struct {
-	Path string `json:"path" yaml:"path"`
-	Wipe bool   `json:"wipe" yaml:"wipe"`
+	Path    string `json:"path" yaml:"path"`
+	Wipe    bool   `json:"wipe" yaml:"wipe"`
+	Encrypt bool   `json:"encrypt" yaml:"encrypt"`
 }
 
 // Disks is a slice of disks

microceph/ceph/osd.go

@@ -2,9 +2,12 @@ package ceph
 
 import (
 	"context"
+	"crypto/rand"
 	"database/sql"
+	"encoding/base64"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -69,15 +72,155 @@ func nextOSD(s *state.State) (int64, error) {
 	}
 }
 
+// setupEncryptedOSD sets up an encrypted OSD on the given disk.
+//
+// Takes a path to the disk device as well as the osd data path and the osd id.
+// Returns the path to the encrypted device and an error if any.
+func setupEncryptedOSD(devicePath string, osdDataPath string, osdID int64) (string, error) {
+	if err := os.Symlink(devicePath, filepath.Join(osdDataPath, "unencrypted")); err != nil {
+		return "", fmt.Errorf("Failed to add unencrypted block symlink: %w", err)
+	}
+
+	// Create a key for the encrypted device
+	key, err := createKey()
+	if err != nil {
+		return "", fmt.Errorf("Key creation error: %w", err)
+	}
+
+	// Store key in ceph key value store
+	if err = storeKey(key, osdID); err != nil {
+		return "", fmt.Errorf("Key store error: %w", err)
+	}
+
+	// Encrypt the device
+	if err = encryptDevice(devicePath, key); err != nil {
+		return "", fmt.Errorf("Failed to encrypt: %w", err)
+	}
+
+	// Open the encrypted device
+	encryptedDevicePath, err := openEncryptedDevice(devicePath, osdID, key)
+	if err != nil {
+		return "", fmt.Errorf("Failed to open: %w", err)
+	}
+	return encryptedDevicePath, nil
+}
+
+// createKey creates a 128 bytes long key for use with LUKS.
+func createKey() ([]byte, error) {
+	// Generate a random data.
+	key := make([]byte, 96)
+	_, err := rand.Read(key)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to generate random key: %w", err)
+	}
+
+	// Encode as base64, this results in 128 bytes.
+	return []byte(base64.StdEncoding.EncodeToString(key)), nil
+}
+
+// encryptDevice encrypts the given device with the given key.
+func encryptDevice(path string, key []byte) error {
+	// Run the cryptsetup command.
+	cmd := exec.Command(
+		"cryptsetup",
+		"--batch-mode",
+		"--key-file", "-",
+		"luksFormat",
+		path)
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("Error in cryptsetup pipe: %s", err)
+	}
+	if _, err = stdin.Write(key); err != nil {
+		return fmt.Errorf("Error writing key to cryptsetup pipe: %s", err)
+	}
+	stdin.Close()
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Failed to luksFormat device: %s, %s, %s", path, err, out)
+	}
+	return nil
+}
+
+// Store the key in the ceph key value store, under a name that derives from the osd id.
+func storeKey(key []byte, osdID int64) error {
+	// Run the ceph config-key set command
+	_, err := processExec.RunCommand("ceph", "config-key", "set", fmt.Sprintf("microceph:osd.%d/key", osdID), string(key))
+	if err != nil {
+		return fmt.Errorf("Failed to store key: %w", err)
+	}
+	return nil
+}
+
+// Open the encrypted device and return its path.
+func openEncryptedDevice(path string, osdID int64, key []byte) (string, error) {
+	// Run the cryptsetup open command, expect key on stdin
+	cmd := exec.Command(
+		"cryptsetup",
+		"--keyfile-size", "128",
+		"--key-file", "-",
+		"luksOpen",
+		path,
+		fmt.Sprintf("luksosd-%d", osdID),
+	)
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return "", fmt.Errorf("Error in cryptsetup pipe: %s", err)
+	}
+	if _, err = stdin.Write(key); err != nil {
+		return "", fmt.Errorf("Error writing key to cryptsetup pipe: %s", err)
+	}
+	stdin.Close()
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf(`Failed to luksOpen: %s, %s, %s
+
+NOTE: OSD Encryption requires a snapd >= 2.59.1
+Verify your version of snapd by running "snap version"
+`, path, err, out)
+	}
+	return fmt.Sprintf("/dev/mapper/luksosd-%d", osdID), nil
+}
+
+// checkEncryptSupport checks if the kernel supports encryption.
+// Checks performed:
+// - Check if the kernel module is loaded.
+// - Check if we have a mapper control file.
+// - Check if we can access /run
+func checkEncryptSupport() error {
+	// Check if we have a mapper
+	if _, err := os.Stat("/dev/mapper/control"); err != nil {
+		return fmt.Errorf("Missing /dev/mapper/control: %w", err)
+	}
+
+	// Check if we have the dm_crypt module
+	inf, err := os.Stat("/sys/module/dm_crypt")
+	if err != nil || inf == nil || !inf.IsDir() {
+		return fmt.Errorf("Missing dm_crypt module: %w", err)
+	}
+
+	// Check if we can list the /run directory; older snapd had an issue with this, https://github.com/snapcore/snapd/pull/12445
+	if _, err = os.ReadDir("/run"); err != nil {
+		return fmt.Errorf("Can't access /run, might need to update snapd to >=2.59.1: %w", err)
+	}
+	return nil
+}
+
 // AddOSD adds an OSD to the cluster, given a device path and a flag for wiping
-func AddOSD(s *state.State, path string, wipe bool) error {
+func AddOSD(s *state.State, path string, wipe bool, encrypt bool) error {
 	revert := revert.New()
 	defer revert.Fail()
 
 	// Validate the path.
 	if !shared.IsBlockdevPath(path) {
 		return fmt.Errorf("Invalid disk path: %s", path)
 	}
+	// Check if we need to support encryption
+	if encrypt {
+		if err := checkEncryptSupport(); err != nil {
+			return fmt.Errorf("Encryption unsupported on this machine: %w", err)
+		}
+	}
 
 	_, _, major, minor, _, _, err := shared.GetFileStat(path)
 	if err != nil {
@@ -162,19 +305,19 @@ func AddOSD(s *state.State, path string, wipe bool) error {
 		return err
 	}
 
-	// if we fail later, make sure we free up the record
+	dataPath := filepath.Join(os.Getenv("SNAP_COMMON"), "data")
+	osdDataPath := filepath.Join(dataPath, "osd", fmt.Sprintf("ceph-%d", nr))
 
+	// if we fail later, make sure we free up the record
 	revert.Add(func() {
+		os.RemoveAll(osdDataPath)
 		s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
 			database.DeleteDisk(ctx, tx, s.Name(), path)
 			return nil
 		})
 	})
 
 	// Create directory.
-	dataPath := filepath.Join(os.Getenv("SNAP_COMMON"), "data")
-	osdDataPath := filepath.Join(dataPath, "osd", fmt.Sprintf("ceph-%d", nr))
-
 	err = os.MkdirAll(osdDataPath, 0700)
 	if err != nil {
 		return fmt.Errorf("Failed to bootstrap monitor: %w", err)
@@ -186,9 +329,18 @@ func AddOSD(s *state.State, path string, wipe bool) error {
 		return fmt.Errorf("Failed to generate OSD keyring: %w", err)
 	}
 
+	var blockPath string
+	if encrypt {
+		blockPath, err = setupEncryptedOSD(path, osdDataPath, nr)
+		if err != nil {
+			return err
+		}
+	} else {
+		blockPath = path
+	}
+
 	// Setup device symlink.
-	err = os.Symlink(path, filepath.Join(osdDataPath, "block"))
-	if err != nil {
+	if err = os.Symlink(blockPath, filepath.Join(osdDataPath, "block")); err != nil {
 		return fmt.Errorf("Failed to add block symlink: %w", err)
 	}
 

microceph/cmd/microceph/disk_add.go

@@ -14,7 +14,8 @@ type cmdDiskAdd struct {
 	common *CmdControl
 	disk   *cmdDisk
 
-	flagWipe bool
+	flagWipe    bool
+	flagEncrypt bool
 }
 
 func (c *cmdDiskAdd) Command() *cobra.Command {
@@ -25,6 +26,7 @@ func (c *cmdDiskAdd) Command() *cobra.Command {
 	}
 
 	cmd.PersistentFlags().BoolVar(&c.flagWipe, "wipe", false, "Wipe the disk prior to use")
+	cmd.PersistentFlags().BoolVar(&c.flagEncrypt, "encrypt", false, "Encrypt the disk prior to use")
 
 	return cmd
 }
@@ -45,8 +47,9 @@ func (c *cmdDiskAdd) Run(cmd *cobra.Command, args []string) error {
 	}
 
 	req := &types.DisksPost{
-		Path: args[0],
-		Wipe: c.flagWipe,
+		Path:    args[0],
+		Wipe:    c.flagWipe,
+		Encrypt: c.flagEncrypt,
 	}
 
 	err = client.AddDisk(context.Background(), cli, req)

microceph/cmd/microceph/init.go

@@ -158,10 +158,16 @@ func (c *cmdInit) Run(cmd *cobra.Command, args []string) error {
 				return err
 			}
 
+			diskEncrypt, err := cli.AskBool("Would you like the disk to be encrypted? [default=no]: ", "no")
+			if err != nil {
+				return err
+			}
+
 			// Add the disk.
 			req := &types.DisksPost{
-				Path: diskPath,
-				Wipe: diskWipe,
+				Path:    diskPath,
+				Wipe:    diskWipe,
+				Encrypt: diskEncrypt,
 			}
 
 			err = client.AddDisk(context.Background(), lc, req)

snapcraft.yaml

@@ -34,6 +34,7 @@ apps:
     daemon: simple
     plugs:
       - block-devices
+      - dm-crypt
       - hardware-observe
       - network
       - network-bind
@@ -70,6 +71,7 @@ apps:
     stop-timeout: 5m
     plugs:
       - block-devices
+      - dm-crypt
       - hardware-observe
       - network
       - network-bind
@@ -96,6 +98,8 @@ apps:
     command: commands/microceph
     plugs:
       - network
+      - block-devices
+      - dm-crypt
   rbd:
     command: commands/rbd
     plugs: