Implement dynamic key persistence

connection_ssh.go

@@ -3,14 +3,6 @@ package fpoc
 import (
 	"context"
 	"crypto"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-
-	"github.com/hashicorp/go-hclog"
-	"golang.org/x/crypto/ssh"
 
 	"gitlab.com/gitlab-org/fleeting/fleeting/provider"
 )
@@ -20,57 +12,12 @@ type PrivPub interface {
 	Public() crypto.PublicKey
 }
 
-// initSSHKey prepare dynamic ssh key for flatcar instances
-func (g *InstanceGroup) initSSHKey(_ context.Context, log hclog.Logger, settings *provider.Settings) error {
-	var key PrivPub
-	var err error
-
-	if len(settings.ConnectorConfig.Key) == 0 {
-		log.Info("Generating dynamic SSH key...")
-
-		key, err = rsa.GenerateKey(rand.Reader, 4096)
-		if err != nil {
-			return fmt.Errorf("generating private key: %w", err)
-		}
-		settings.ConnectorConfig.Key = pem.EncodeToMemory(
-			&pem.Block{
-				Type:  "RSA PRIVATE KEY",
-				Bytes: x509.MarshalPKCS1PrivateKey(key.(*rsa.PrivateKey)),
-			},
-		)
-
-		log.Debug("Key generated")
-	} else {
-		var ok bool
-
-		priv, err := ssh.ParseRawPrivateKey(settings.ConnectorConfig.Key)
-		if err != nil {
-			return fmt.Errorf("reading private key: %w", err)
-		}
-
-		key, ok = priv.(PrivPub)
-		if !ok {
-			return fmt.Errorf("key doesn't export PublicKey()")
-		}
-	}
-
-	log.Debug("Extracting public key...")
-	sshPubKey, err := ssh.NewPublicKey(key.Public())
+// ssh handles non static ssh keys
+func (g *InstanceGroup) ssh(ctx context.Context, info provider.ConnectInfo) error {
+	privateKeyPem, _, err := GetInstanceSSHKey(g.settings, info.ID, g.SSHStoragePath)
 	if err != nil {
-		return fmt.Errorf("generating private key: %w", err)
-	}
-
-	g.sshPubKey = string(ssh.MarshalAuthorizedKey(sshPubKey))
-	log.With("public_key", g.sshPubKey).Debug("Extracted public key")
-
-	if g.imgProps != nil {
-		if g.imgProps.OSAdminUser == "" && settings.Username == "" {
-			return fmt.Errorf("image properties 'os_admin_user' and 'runners.autoscaler.connector_config.username' missing. Ensure one is set.")
-		}
-		if g.imgProps.OSAdminUser != "" && settings.Username == "" {
-			settings.Username = g.imgProps.OSAdminUser
-		}
+		return err
 	}
-
+	info.Key = privateKeyPem
 	return nil
 }

provider.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"path"
+	"path/filepath"
 	"sync/atomic"
 	"time"
 
@@ -27,14 +29,14 @@ type InstanceGroup struct {
 	NovaMicroversion string        `json:"nova_microversion"` // Microversion for the Nova client
 	ServerSpec       ExtCreateOpts `json:"server_spec"`       // instance creation spec
 	UseIgnition      bool          `json:"use_ignition"`      // Configure keys via Ignition (Fedora CoreOS / Flatcar)
+	SSHStoragePath   string        `json:"storage_path"`      // Path to storage dynamic ssh keys in
 	BootTimeS        string        `json:"boot_time"`         // optional: wait some time before report machine as available
 	BootTime         time.Duration
 
 	client          openstackclient.Client
 	settings        provider.Settings
 	log             hclog.Logger
 	imgProps        *openstackclient.ImageProperties
-	sshPubKey       string
 	instanceCounter atomic.Int32
 }
 
@@ -76,9 +78,13 @@ func (g *InstanceGroup) Init(ctx context.Context, log hclog.Logger, settings pro
 	}
 
 	if g.UseIgnition {
-		err = g.initSSHKey(ctx, log, &settings)
-		if err != nil {
-			return provider.ProviderInfo{}, err
+		if g.imgProps != nil {
+			if g.imgProps.OSAdminUser == "" && settings.Username == "" {
+				return provider.ProviderInfo{}, fmt.Errorf("image properties 'os_admin_user' and 'runners.autoscaler.connector_config.username' missing. Ensure one is set")
+			}
+			if g.imgProps.OSAdminUser != "" && settings.Username == "" {
+				settings.Username = g.imgProps.OSAdminUser
+			}
 		}
 	}
 
@@ -189,6 +195,13 @@ func (g *InstanceGroup) Decrease(ctx context.Context, instances []string) (succe
 			g.log.Info("Instance deletion request successful", "id", id)
 			succeeded = append(succeeded, id)
 		}
+
+		// Delete dynamic ssh key if persistence is enabled
+		if g.SSHStoragePath != "" {
+			err3 := os.Remove(filepath.Join(g.SSHStoragePath, id))
+			g.log.Error("failed to delete dynamic sshkey file: %w", err3)
+			err = errors.Join(err, err3)
+		}
 	}
 
 	g.log.Info("Decrease", "instances", instances)
@@ -217,14 +230,17 @@ func (g *InstanceGroup) getInstances(ctx context.Context) ([]servers.Server, err
 
 func (g *InstanceGroup) createInstance(ctx context.Context) (string, error) {
 	spec := new(ExtCreateOpts)
+
+	// Initialize the server spec with user provided configuration
 	err := copier.Copy(spec, &g.ServerSpec)
 	if err != nil {
 		return "", err
 	}
 
 	index := int(g.instanceCounter.Add(1))
+	instanceName := fmt.Sprintf(g.ServerSpec.Name, index)
 
-	spec.Name = fmt.Sprintf(g.ServerSpec.Name, index)
+	spec.Name = instanceName
 	if spec.Metadata == nil {
 		spec.Metadata = make(map[string]string)
 	}
@@ -236,7 +252,11 @@ func (g *InstanceGroup) createInstance(ctx context.Context) (string, error) {
 	}
 
 	if g.UseIgnition {
-		err := InsertSSHKeyIgn(spec, g.settings.Username, g.sshPubKey)
+		_, publicKeyPem, err := GetInstanceSSHKey(g.settings, instanceName, g.SSHStoragePath)
+		if err != nil {
+			return "", err
+		}
+		err = InsertSSHKeyIgn(spec, g.settings.Username, string(publicKeyPem))
 		if err != nil {
 			return "", err
 		}
@@ -253,10 +273,9 @@ func (g *InstanceGroup) createInstance(ctx context.Context) (string, error) {
 func (g *InstanceGroup) ConnectInfo(ctx context.Context, instanceID string) (provider.ConnectInfo, error) {
 	srv, err := g.client.GetServer(ctx, instanceID)
 	if err != nil {
-		return provider.ConnectInfo{}, fmt.Errorf("Failed to get server %s: %w", instanceID, err)
+		return provider.ConnectInfo{}, fmt.Errorf("failed to get server %s: %w", instanceID, err)
 	}
 
-	// g.log.Debug("Server info", "srv", srv)
 	if srv.Status != "ACTIVE" {
 		return provider.ConnectInfo{}, fmt.Errorf("instance status is not active: %s", srv.Status)
 	}
@@ -278,11 +297,14 @@ func (g *InstanceGroup) ConnectInfo(ctx context.Context, instanceID string) (pro
 	}
 
 	info := provider.ConnectInfo{
-		ConnectorConfig: g.settings.ConnectorConfig,
-		ID:              instanceID,
-		InternalAddr:    ipAddr,
-		ExternalAddr:    ipAddr,
+		ID:           instanceID,
+		InternalAddr: ipAddr,
+		ExternalAddr: ipAddr,
 	}
+
+	// We might inject ConnectorConfig.Key with a dynamic instance key, don't polute the global ConnectorConfig instance.
+	copier.Copy(info.ConnectorConfig, &g.settings.ConnectorConfig)
+
 	info.Protocol = provider.ProtocolSSH
 
 	if g.imgProps != nil {
@@ -317,8 +339,15 @@ func (g *InstanceGroup) ConnectInfo(ctx context.Context, instanceID string) (pro
 		info.OS = "linux"
 		info.Arch = "amd64"
 	}
+	if info.UseStaticCredentials {
+		return info, nil
+	}
 
-	return info, nil
+	switch info.Protocol {
+	case provider.ProtocolSSH:
+		err = g.ssh(ctx, info)
+	}
+	return info, err
 }
 
 func (g *InstanceGroup) Shutdown(ctx context.Context) error {

utils.go

@@ -1,17 +1,28 @@
 package fpoc
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
+	"errors"
 	"fmt"
 	"maps"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
+	"golang.org/x/crypto/ssh"
+
 	igncfg "github.com/coreos/ignition/v2/config/v3_4"
 	igntyp "github.com/coreos/ignition/v2/config/v3_4/types"
 	"github.com/coreos/vcontext/report"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
 	"github.com/mitchellh/mapstructure"
+
+	"gitlab.com/gitlab-org/fleeting/fleeting/provider"
 )
 
 // ExtCreateOpts extended version of servers.CreateOpts
@@ -189,3 +200,89 @@ func InsertSSHKeyIgn(spec *ExtCreateOpts, username, pubKey string) error {
 	spec.UserData = string(buf)
 	return nil
 }
+
+func CheckFileExists(filePath string) bool {
+	_, error := os.Stat(filePath)
+	return !errors.Is(error, os.ErrNotExist)
+}
+
+func GetInstanceSSHKey(settings provider.Settings, instanceId string, storagePath string) (privateKeyPem []byte, publicKeyPem []byte, err error) {
+	if settings.UseStaticCredentials && storagePath != "" {
+		return nil, nil, fmt.Errorf("storage_path must be empty when using static credentials")
+	}
+
+	if settings.UseStaticCredentials && len(settings.ConnectorConfig.Key) == 0 {
+		return nil, nil, fmt.Errorf("key must be provided when using static credentials")
+	}
+
+	var privateKey PrivPub
+	instanceSSHKeyFile := filepath.Join(storagePath, instanceId)
+	instanceSSHKeyFileExists := CheckFileExists(instanceSSHKeyFile)
+
+	if settings.UseStaticCredentials && len(settings.ConnectorConfig.Key) != 0 {
+		// Use static key provided by runner configuration
+		privateKey, err = ParseRawPrivateKey(settings.ConnectorConfig.Key)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to parse key: %w", err)
+		}
+	} else if storagePath != "" && instanceSSHKeyFileExists {
+		// Use pre-generated dynamic instance key
+		plainKey, err := os.ReadFile(instanceSSHKeyFile)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to read key file: %w", err)
+		}
+
+		privateKey, err = ParseRawPrivateKey(plainKey)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to parse key: %w", err)
+		}
+	} else {
+		// Generate dynamic instance key
+		privateKey, err = rsa.GenerateKey(rand.Reader, 4096)
+		if err != nil {
+			return nil, nil, fmt.Errorf("generating private key failed: %w", err)
+		}
+	}
+
+	privateKeyPem, publicKeyPem, err = GenerateSSHKeyPem(privateKey)
+
+	if storagePath != "" && !instanceSSHKeyFileExists {
+		err = os.WriteFile(instanceSSHKeyFile, privateKeyPem, 0600)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to write key file: %w", err)
+		}
+	}
+
+	return privateKeyPem, publicKeyPem, err
+}
+
+func ParseRawPrivateKey(key []byte) (privateKey PrivPub, err error) {
+	pkey, err := ssh.ParseRawPrivateKey(key)
+	if err != nil {
+		return nil, fmt.Errorf("reading private key failed: %w", err)
+	}
+
+	var ok bool
+	privateKey, ok = pkey.(PrivPub)
+	if !ok {
+		return nil, fmt.Errorf("key doesn't export PublicKey()")
+	}
+
+	return privateKey, nil
+}
+func GenerateSSHKeyPem(key PrivPub) (privateKeyPem []byte, publicKeyPem []byte, err error) {
+	privateKeyPem = pem.EncodeToMemory(
+		&pem.Block{
+			Type:  "RSA PRIVATE KEY",
+			Bytes: x509.MarshalPKCS1PrivateKey(key.(*rsa.PrivateKey)),
+		},
+	)
+
+	publicKey, err := ssh.NewPublicKey(key.Public())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return privateKeyPem, ssh.MarshalAuthorizedKey(publicKey), nil
+
+}