Skip to content

Commit

Permalink
Removed ceph.conf parsing and replaced with calls to ceph(-conf)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MadnessASAP
MadnessASAP committed Dec 10, 2024
1 parent 586c56a commit 4dd609e
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 177 deletions.
8 changes: 0 additions & 8 deletions internal/server/metadata/configuration.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,6 @@
"type": "string"
}
},
{
"ceph.user_key": {
"required": "no",
"type": "string",
"shortdesc": "Ceph user key",
"longdesc": "Usually inferred using Ceph client tools on the host."
}
},
{
"initial.*": {
"longdesc": "",
Expand Down
236 changes: 67 additions & 169 deletions internal/server/storage/drivers/utils_ceph.go
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,34 @@
package drivers

import (
"bufio"
"encoding/json"
"fmt"
"os"
"strings"

"github.com/lxc/incus/v6/shared/api"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/subprocess"
"github.com/lxc/incus/v6/shared/util"
)

// CephMonmap represents the json (partial) output of `ceph mon dump`.
type CephMonmap struct {
Fsid string `json:"fsid"`
Mons []struct {
Name string `json:"name"`
PublicAddrs struct {
Addrvec []struct {
Type string `json:"type"`
Addr string `json:"addr"`
Nonce int `json:"nonce"`
} `json:"addrvec"`
} `json:"public_addrs"`
Addr string `json:"addr"`
PublicAddr string `json:"public_addr"`
Priority int `json:"priority"`
Weight int `json:"weight"`
} `json:"mons"`
}

// CephGetRBDImageName returns the RBD image name as it is used in ceph.
// Example:
// A custom block volume named vol1 in project default will return custom_default_vol1.block.
Expand Down Expand Up @@ -61,191 +79,71 @@ func CephGetRBDImageName(vol Volume, snapName string, zombie bool) string {

// CephMonitors gets the mon-host field for the relevant cluster and extracts the list of addresses and ports.
func CephMonitors(cluster string) ([]string, error) {
// Open the CEPH configuration.
cephConf, err := os.Open(fmt.Sprintf("/etc/ceph/%s.conf", cluster))
if err != nil {
return nil, fmt.Errorf("Failed to open %q: %w", fmt.Sprintf("/etc/ceph/%s.conf", cluster), err)
}

// Locate the mon-host key and its values.
ctx := logger.Ctx{"cluster": cluster}
cephMon := []string{}
scan := bufio.NewScanner(cephConf)
for scan.Scan() {
line := scan.Text()
line = strings.TrimSpace(line)

if line == "" {
continue
}

if strings.HasPrefix(line, "mon_host") || strings.HasPrefix(line, "mon-host") || strings.HasPrefix(line, "mon host") {
fields := strings.SplitN(line, "=", 2)
if len(fields) < 2 {
continue
}

// Parsing mon_host is quite tricky.
// It supports a space separate list of comma separated lists of:
// - DNS names
// - IPv4 addresses
// - IPv6 addresses (square brackets)
// - Optional version indicator
// - Optional port numbers
// - Optional data (after / separator)
// - Tuples of addresses with all the above still applying inside the tuple
//
// As this function is primarily used for cephfs which
// doesn't take the version indication, trailing bits or supports those
// tuples, all of those effectively get stripped away to get a clean
// address list (with ports).
entries := strings.Split(fields[1], " ")
for _, entry := range entries {
servers := strings.Split(entry, ",")
for _, server := range servers {
// Trim leading/trailing spaces.
server = strings.TrimSpace(server)

// Trim leading protocol version.
server = strings.TrimPrefix(server, "v1:")
server = strings.TrimPrefix(server, "v2:")
server = strings.TrimPrefix(server, "[v1:")
server = strings.TrimPrefix(server, "[v2:")

// Trim trailing divider.
server = strings.Split(server, "/")[0]

// Handle end of nested blocks.
server = strings.ReplaceAll(server, "]]", "]")
if !strings.HasPrefix(server, "[") {
server = strings.TrimSuffix(server, "]")
}

// Trim any spaces.
server = strings.TrimSpace(server)

// If nothing left, skip.
if server == "" {
continue
}

// Append the default v1 port if none are present.
if !strings.HasSuffix(server, ":6789") && !strings.HasSuffix(server, ":3300") {
server += ":6789"
}

cephMon = append(cephMon, strings.TrimSpace(server))
}
// First try the ceph client admin tool
jsonBlob, err := subprocess.RunCommand(
"ceph", "mon", "dump",
"--cluster", cluster,
"--format", "json",
)
if err != nil {
ctx["err"] = err
logger.Warn("Failed to get monitors from ceph client", ctx)
} else {
// Extract monitor public addresses from json
var monmap CephMonmap
err = json.Unmarshal([]byte(jsonBlob), &monmap)
if err != nil {
ctx["err"] = err
logger.Warn("Failed to decode json from 'ceph mon dump'", ctx)
} else {
for _, mon := range monmap.Mons {
cephMon = append(cephMon, mon.PublicAddr)
}
}
}

// If that fails try ceph-conf
if len(cephMon) == 0 {
return nil, fmt.Errorf("Couldn't find a CEPH mon")
}

return cephMon, nil
}

func getCephKeyFromFile(path string) (string, error) {
cephKeyring, err := os.Open(path)
if err != nil {
return "", fmt.Errorf("Failed to open %q: %w", path, err)
}

// Locate the keyring entry and its value.
var cephSecret string
scan := bufio.NewScanner(cephKeyring)
for scan.Scan() {
line := scan.Text()
line = strings.TrimSpace(line)

if line == "" {
continue
}

if strings.HasPrefix(line, "key") {
fields := strings.SplitN(line, "=", 2)
if len(fields) < 2 {
continue
monBlob, err := subprocess.RunCommand(
"ceph-conf", "mon_host",
"--cluster", cluster,
)
if err != nil {
logger.Warn("Failed to get monitors from ceph-conf", logger.Ctx{"cluster": cluster, "err": err})
} else {
// sadly ceph-conf doesn't give us a nice json blob like 'ceph mon dump'
for _, mon := range strings.Split(monBlob, " ") {
mon = strings.Trim(mon, "[]")
for _, addr := range strings.Split(mon, ",") {
addr = strings.Split(addr, ":")[1]
addr = strings.Split(addr, "/")[0]
cephMon = append(cephMon, addr)
}
}

cephSecret = strings.TrimSpace(fields[1])
break
}
}

if cephSecret == "" {
return "", fmt.Errorf("Couldn't find a keyring entry")
if len(cephMon) == 0 {
return nil, fmt.Errorf("Failed to retrieve monitors for %q", cluster)
}

return cephSecret, nil
return cephMon, nil
}

// CephKeyring gets the key for a particular Ceph cluster and client name.
func CephKeyring(cluster string, client string) (string, error) {
var cephSecret string
cephConfigPath := fmt.Sprintf("/etc/ceph/%v.conf", cluster)

keyringPathFull := fmt.Sprintf("/etc/ceph/%v.client.%v.keyring", cluster, client)
keyringPathCluster := fmt.Sprintf("/etc/ceph/%v.keyring", cluster)
keyringPathGlobal := "/etc/ceph/keyring"
keyringPathGlobalBin := "/etc/ceph/keyring.bin"

if util.PathExists(keyringPathFull) {
return getCephKeyFromFile(keyringPathFull)
} else if util.PathExists(keyringPathCluster) {
return getCephKeyFromFile(keyringPathCluster)
} else if util.PathExists(keyringPathGlobal) {
return getCephKeyFromFile(keyringPathGlobal)
} else if util.PathExists(keyringPathGlobalBin) {
return getCephKeyFromFile(keyringPathGlobalBin)
} else if util.PathExists(cephConfigPath) {
// Open the CEPH config file.
cephConfig, err := os.Open(cephConfigPath)
if err != nil {
return "", fmt.Errorf("Failed to open %q: %w", cephConfigPath, err)
}

// Locate the keyring entry and its value.
scan := bufio.NewScanner(cephConfig)
for scan.Scan() {
line := scan.Text()
line = strings.TrimSpace(line)

if line == "" {
continue
}

if strings.HasPrefix(line, "key") {
fields := strings.SplitN(line, "=", 2)
if len(fields) < 2 {
continue
}

// Check all key related config keys.
switch strings.TrimSpace(fields[0]) {
case "key":
cephSecret = strings.TrimSpace(fields[1])
case "keyfile":
key, err := os.ReadFile(fields[1])
if err != nil {
return "", err
}

cephSecret = strings.TrimSpace(string(key))
case "keyring":
return getCephKeyFromFile(strings.TrimSpace(fields[1]))
}
}

if cephSecret != "" {
break
}
}
}

if cephSecret == "" {
return "", fmt.Errorf("Couldn't find a keyring entry")
cephSecret, err := subprocess.RunCommand(
"ceph-conf", "key",
"--cluster", cluster,
"--name", "client."+client,
)
if err != nil {
return "", fmt.Errorf("Failed to get key for 'client.%s' on %q", client, cluster)
}

return cephSecret, nil
Expand Down

0 comments on commit 4dd609e

Please sign in to comment.