Skip to content

Commit

Permalink
Merge pull request rancher#44977 from slickwarren/cwarren/v2.9/valida…
Browse files Browse the repository at this point in the history 
…te-cert-rotation

[v2.9] add rke1 to cert rotation test. Add validation to cert rotation.
  • Loading branch information
@slickwarren
slickwarren authored Apr 10, 2024
2 parents 86f85b2 + 07220dd commit 46936e5
Show file tree
Hide file tree
Showing 2 changed files with 317 additions and 12 deletions.
289 changes: 281 additions & 8 deletions tests/v2/validation/certrotation/cert_rotation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,39 @@ package certrotation

import (
"context"
"errors"
"io"
"net/http"
"regexp"
"strings"

"github.com/rancher/norman/types"
apiv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
rkev1 "github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1"
"github.com/rancher/shepherd/clients/rancher"
management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"
v1 "github.com/rancher/shepherd/clients/rancher/v1"
"github.com/rancher/shepherd/extensions/clusters"
"github.com/rancher/shepherd/extensions/defaults"
"github.com/rancher/shepherd/extensions/provisioning"
"github.com/rancher/shepherd/extensions/sshkeys"
"github.com/rancher/shepherd/pkg/nodes"
"github.com/rancher/shepherd/pkg/wait"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
namespace = "fleet-default"
ProvisioningSteveResouceType = "provisioning.cattle.io.cluster"
machineSteveResourceType = "cluster.x-k8s.io.machine"
machineSteveAnnotation = "cluster.x-k8s.io/machine"
etcdLabel = "node-role.kubernetes.io/etcd"
clusterLabel = "cluster.x-k8s.io/cluster-name"
namespace = "fleet-default"
provisioningSteveResourceType = "provisioning.cattle.io.cluster"
machineSteveResourceType = "cluster.x-k8s.io.machine"
machineSteveAnnotation = "cluster.x-k8s.io/machine"
etcdLabel = "node-role.kubernetes.io/etcd"
clusterLabel = "cluster.x-k8s.io/cluster-name"
certFileExtension = ".crt"
pemFileExtension = ".pem"

privateKeySSHKeyRegExPattern = `-----BEGIN RSA PRIVATE KEY-{3,}\n([\s\S]*?)\n-{3,}END RSA PRIVATE KEY-----`
)

// rotateCerts rotates the certificates in a cluster
Expand All @@ -41,7 +54,7 @@ func rotateCerts(client *rancher.Client, clusterName string) error {
return err
}

cluster, err := adminClient.Steve.SteveType(ProvisioningSteveResouceType).ByID(id)
cluster, err := adminClient.Steve.SteveType(provisioningSteveResourceType).ByID(id)
if err != nil {
return err
}
Expand All @@ -52,6 +65,40 @@ func rotateCerts(client *rancher.Client, clusterName string) error {
return err
}

clusterID, err := clusters.GetClusterIDByName(client, clusterName)
if err != nil {
return err
}

steveclient, err := client.Steve.ProxyDownstream(clusterID)
if err != nil {
return err
}

nodeList, err := steveclient.SteveType("node").List(nil)
if err != nil {
return err
}

nodeCertificates := map[string]map[string]string{}

sshUser, err := sshkeys.GetSSHUser(client, cluster)
if err != nil {
return err
}
if sshUser == "" {
return errors.New("sshUser does not exist")
}

for _, node := range nodeList.Data {
newCertificate, err := getCertificatesFromMachine(client, &node, sshUser)
if err != nil {
return err
}

nodeCertificates[node.ID] = newCertificate
}

updatedCluster := *cluster
generation := int64(1)
if clusterSpec.RKEConfig.RotateCertificates != nil {
Expand All @@ -64,7 +111,7 @@ func rotateCerts(client *rancher.Client, clusterName string) error {

updatedCluster.Spec = *clusterSpec

_, err = client.Steve.SteveType(ProvisioningSteveResouceType).Update(cluster, updatedCluster)
_, err = client.Steve.SteveType(provisioningSteveResourceType).Update(cluster, updatedCluster)
if err != nil {
return err
}
Expand Down Expand Up @@ -104,5 +151,231 @@ func rotateCerts(client *rancher.Client, clusterName string) error {
if err != nil {
return err
}

postRotatedCertificates := map[string]map[string]string{}

for _, node := range nodeList.Data {
newCertificate, err := getCertificatesFromMachine(client, &node, sshUser)
if err != nil {
return err
}

postRotatedCertificates[node.ID] = newCertificate
}

isAllCertRotated := compareCertificatesFromMachines(nodeCertificates, postRotatedCertificates)
if !isAllCertRotated {
return errors.New("certs weren't properly rotated")
}
return nil
}

func rotateRKE1Certs(client *rancher.Client, clusterName string) error {
clusterID, err := clusters.GetClusterIDByName(client, clusterName)
if err != nil {
return err
}

nodeList, err := client.Management.Node.List(&types.ListOpts{Filters: map[string]interface{}{
"clusterId": clusterID,
}})
if err != nil {
return err
}

nodeCertificates := map[string]map[string]string{}

for _, node := range nodeList.Data {
newCertificate, err := getCertificatesFromV3Node(client, &node)
if err != nil {
return err
}

nodeCertificates[node.ID] = newCertificate
}

cluster, err := client.Management.Cluster.ByID(clusterID)
if err != nil {
return err
}

_, err = client.Management.Cluster.ActionRotateCertificates(
cluster,
&management.RotateCertificateInput{CACertificates: false, Services: ""},
)
if err != nil {
return err
}

logrus.Infof("updated cluster, certs are rotating...")

adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session)
if err != nil {
return err
}

err = clusters.WaitClusterToBeUpgraded(adminClient, clusterID)
if err != nil {
return err
}

postRotatedCertificates := map[string]map[string]string{}

for _, node := range nodeList.Data {
newCertificate, err := getCertificatesFromV3Node(client, &node)
if err != nil {
return err
}

postRotatedCertificates[node.ID] = newCertificate
}

isAllCertRotated := compareCertificatesFromMachines(nodeCertificates, postRotatedCertificates)
if !isAllCertRotated {
return errors.New("certs weren't properly rotated")
}

return nil
}

func compareCertificatesFromMachines(certObject1, certObject2 map[string]map[string]string) bool {
isRotated := true
for nodeID := range certObject1 {
for certType := range certObject1[nodeID] {

if certObject1[nodeID][certType] == certObject2[nodeID][certType] {
logrus.Infof("non-rotated cert info:")
logrus.Infof("%s %s was not updated: %s", nodeID, certType, certObject1[nodeID][certType])

isRotated = false
}
}
}
return isRotated
}

func getCertificatesFromMachine(client *rancher.Client, machineNode *v1.SteveAPIObject, sshUser string) (map[string]string, error) {
certificates := map[string]string{}

sshNode, err := sshkeys.GetSSHNodeFromMachine(client, sshUser, machineNode)
if err != nil {
return nil, err
}

logrus.Infof("Getting certificates from machine: %s", machineNode.Name)

clusterType := machineNode.Labels["node.kubernetes.io/instance-type"]
certsPath := "/var/lib/rancher/" + clusterType + "/server/tls/"

certsList := []string{
"client-admin",
"client-auth-proxy",
"client-controller",
"client-kube-apiserver",
"client-kubelet",
"client-kube-proxy",

"client-" + clusterType + "-cloud-controller",
"client-" + clusterType + "-controller",

"client-scheduler",
"client-supervisor",
"etcd/client",
"etcd/peer-server-client",
"kube-controller-manager/kube-controller-manager",
"kube-scheduler/kube-scheduler",
"serving-kube-apiserver",
}

for _, filename := range certsList {
// ignoring errors since node roles have different subsets of the certs.
certString, _ := sshNode.ExecuteCommand("openssl x509 -enddate -noout -in " + certsPath + filename + certFileExtension)

if certString != "" {
certificates[filename] = certString
}
}

return certificates, nil
}

func getCertificatesFromV3Node(client *rancher.Client, v3Node *management.Node) (map[string]string, error) {
certificates := map[string]string{}

sshNode, err := getSSHNodeFromV3Node(client, v3Node)
if err != nil {
return nil, err
}

logrus.Infof("Getting certificates from machine: %s", v3Node.ID)

certsPath := "/etc/kubernetes/ssl/"

certsList := []string{
"kube-apiserver",
"kube-apiserver-proxy-client",
"$(find /etc/kubernetes/ssl -name 'kube-etcd-*' | grep -v \"key\")",
"kube-controller-manager",
"kube-node",
"kube-proxy",
"kube-scheduler",
}

for _, filename := range certsList {
fullAbsolutePath := certsPath + filename + pemFileExtension
if strings.Contains(filename, "etcd") {
fullAbsolutePath = filename
}

certString, _ := sshNode.ExecuteCommand("openssl x509 -enddate -noout -in " + fullAbsolutePath)

if certString != "" {
certificates[filename] = certString
}
}

return certificates, nil
}

func downloadRKE1SshKeys(client *rancher.Client, v3Node *management.Node) ([]byte, error) {
sshKeyLink := v3Node.Links["nodeConfig"]

req, err := http.NewRequest("GET", sshKeyLink, nil)
if err != nil {
return nil, err
}

req.Header.Add("Authorization", "Bearer "+client.RancherConfig.AdminToken)

resp, err := client.Management.APIBaseClient.Ops.Client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()

bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}

privateSSHKeyRegEx := regexp.MustCompile(privateKeySSHKeyRegExPattern)
privateSSHKey := privateSSHKeyRegEx.FindString(string(bodyBytes))

return []byte(privateSSHKey), err
}

func getSSHNodeFromV3Node(client *rancher.Client, v3Node *management.Node) (*nodes.Node, error) {
sshkey, err := downloadRKE1SshKeys(client, v3Node)
if err != nil {
return nil, err
}

clusterNode := &nodes.Node{
NodeID: v3Node.ID,
PublicIPAddress: v3Node.ExternalIPAddress,
SSHUser: v3Node.SshUser,
SSHKey: sshkey,
}

return clusterNode, nil
}
40 changes: 36 additions & 4 deletions tests/v2/validation/certrotation/cert_rotation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@
package certrotation

import (
"strings"
"testing"

provv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
"github.com/rancher/shepherd/clients/rancher"
steveV1 "github.com/rancher/shepherd/clients/rancher/v1"
"github.com/rancher/shepherd/extensions/clusters"
"github.com/rancher/shepherd/extensions/provisioninginput"
"github.com/rancher/shepherd/pkg/config"
"github.com/rancher/shepherd/pkg/session"
Expand Down Expand Up @@ -38,10 +42,38 @@ func (r *V2ProvCertRotationTestSuite) SetupSuite() {
}

func (r *V2ProvCertRotationTestSuite) TestCertRotation() {
r.Run("test-cert-rotation", func() {
require.NoError(r.T(), rotateCerts(r.client, r.client.RancherConfig.ClusterName))
require.NoError(r.T(), rotateCerts(r.client, r.client.RancherConfig.ClusterName))
})
id, err := clusters.GetV1ProvisioningClusterByName(r.client, r.client.RancherConfig.ClusterName)
require.NoError(r.T(), err)

cluster, err := r.client.Steve.SteveType(provisioningSteveResourceType).ByID(id)
require.NoError(r.T(), err)

spec := &provv1.ClusterSpec{}
err = steveV1.ConvertToK8sType(cluster.Spec, spec)
require.NoError(r.T(), err)

clusterType := "RKE1"

if strings.Contains(spec.KubernetesVersion, "-rancher") || len(spec.KubernetesVersion) == 0 {
r.Run("test-cert-rotation "+clusterType, func() {
require.NoError(r.T(), rotateRKE1Certs(r.client, r.client.RancherConfig.ClusterName))
require.NoError(r.T(), rotateRKE1Certs(r.client, r.client.RancherConfig.ClusterName))
})

} else {

if strings.Contains(spec.KubernetesVersion, "k3s") {
clusterType = "K3s"
} else {
clusterType = "RKE2"
}

r.Run("test-cert-rotation "+clusterType, func() {
require.NoError(r.T(), rotateCerts(r.client, r.client.RancherConfig.ClusterName))
require.NoError(r.T(), rotateCerts(r.client, r.client.RancherConfig.ClusterName))
})
}

}

func TestCertRotationTestSuite(t *testing.T) {
Expand Down

0 comments on commit 46936e5

Please sign in to comment.