Skip to content

Commit

Permalink
Test template-validator certificate rotation
Browse files Browse the repository at this point in the history
The test checks the TLS certificate returned
by the template validator webhook server.

The port-forwarding code was mostly inspired
by the implementation in kubectl.

Signed-off-by: Andrej Krejcir <[email protected]>
  • Loading branch information
akrejcir authored and kubevirt-bot committed May 7, 2021
1 parent e123f86 commit 518d02e
Show file tree
Hide file tree
Showing 29 changed files with 4,901 additions and 13 deletions.
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docker/libnetwork v0.0.0-20190731215715-7f13a5c99f4b/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s=
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
Expand All @@ -234,6 +235,7 @@ github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6
github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
Expand Down
8 changes: 4 additions & 4 deletions internal/operands/template-validator/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// repository, and import it as a go module

const (
containerPort = 8443
ContainerPort = 8443
KubevirtIo = "kubevirt.io"
SecretName = "virt-template-validator-certs"
VirtTemplateValidator = "virt-template-validator"
Expand Down Expand Up @@ -101,7 +101,7 @@ func newService(namespace string) *core.Service {
Ports: []core.ServicePort{{
Name: "webhook",
Port: 443,
TargetPort: intstr.FromInt(containerPort),
TargetPort: intstr.FromInt(ContainerPort),
}},
Selector: commonLabels(),
},
Expand Down Expand Up @@ -139,7 +139,7 @@ func newDeployment(namespace string, replicas int32, image string) *apps.Deploym
ImagePullPolicy: core.PullAlways,
Args: []string{
"-v=2",
fmt.Sprintf("--port=%d", containerPort),
fmt.Sprintf("--port=%d", ContainerPort),
fmt.Sprintf("--cert-dir=%s", certMountPath),
},
VolumeMounts: []core.VolumeMount{{
Expand All @@ -152,7 +152,7 @@ func newDeployment(namespace string, replicas int32, image string) *apps.Deploym
},
Ports: []core.ContainerPort{{
Name: "webhook",
ContainerPort: containerPort,
ContainerPort: ContainerPort,
Protocol: core.ProtocolTCP,
}},
}},
Expand Down
106 changes: 106 additions & 0 deletions tests/port-forwarding.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package tests

import (
"fmt"
"io"
"net"
"net/http"
"strconv"
"sync/atomic"

. "github.com/onsi/ginkgo"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/httpstream"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/portforward"
"k8s.io/client-go/transport/spdy"
)

type PortForwarder interface {
Connect(pod *core.Pod, remotePort uint16) (net.Conn, error)
}

type portForwarderImpl struct {
config *rest.Config
client rest.Interface
requestId int32
}

var _ PortForwarder = &portForwarderImpl{}

func (p *portForwarderImpl) Connect(pod *core.Pod, remotePort uint16) (net.Conn, error) {
streamConnection, err := p.createStreamConnection(pod)
if err != nil {
return nil, err
}

requestId := atomic.AddInt32(&p.requestId, 1)

// Error stream is needed, otherwise port-forwarding will not work
headers := http.Header{}
headers.Set(core.StreamType, core.StreamTypeError)
headers.Set(core.PortHeader, fmt.Sprintf("%d", remotePort))
headers.Set(core.PortForwardRequestIDHeader, strconv.Itoa(int(requestId)))
errorStream, err := streamConnection.CreateStream(headers)
if err != nil {
streamConnection.Close()
return nil, err
}
// We will not write to error stream
errorStream.Close()

headers.Set(core.StreamType, core.StreamTypeData)
dataStream, err := streamConnection.CreateStream(headers)
if err != nil {
streamConnection.Close()
return nil, err
}

pipeIn, pipeOut := net.Pipe()
// Read data from pod
go func() {
defer pipeIn.Close()
_, err := io.Copy(pipeIn, dataStream)
if err != nil {
fmt.Fprintf(GinkgoWriter, "Error reading from port-forwarding: %v", err)
return
}
}()

// Send data to pod
go func() {
defer streamConnection.Close()
defer dataStream.Close()
_, err := io.Copy(dataStream, pipeIn)
if err != nil {
fmt.Fprintf(GinkgoWriter, "Error writing to port-forwarding: %v", err)
return
}
}()

return pipeOut, nil
}

func (p *portForwarderImpl) createStreamConnection(pod *core.Pod) (httpstream.Connection, error) {
transport, upgrader, err := spdy.RoundTripperFor(p.config)
if err != nil {
return nil, err
}

req := p.client.Post().
Resource("pods").
Namespace(pod.Namespace).
Name(pod.Name).
SubResource("portforward")

dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", req.URL())
streamConn, _, err := dialer.Dial(portforward.PortForwardProtocolV1Name)
return streamConn, err
}

func NewPortForwarder(config *rest.Config, client rest.Interface) PortForwarder {
return &portForwarderImpl{
config: config,
client: client,
}
}
3 changes: 3 additions & 0 deletions tests/tests_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ var (
ctx context.Context
strategy TestSuiteStrategy
sspListerWatcher cache.ListerWatcher
portForwarder PortForwarder
deploymentTimedOut bool
)

Expand Down Expand Up @@ -324,6 +325,8 @@ func setupApiClient() {
coreClient, err = kubernetes.NewForConfig(cfg)
Expect(err).ToNot(HaveOccurred())

portForwarder = NewPortForwarder(cfg, coreClient.CoreV1().RESTClient())

ctx = context.Background()
sspListerWatcher = createSspListerWatcher(cfg)
}
Expand Down
64 changes: 55 additions & 9 deletions tests/validator_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package tests

import (
"crypto/tls"
"crypto/x509"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -721,19 +723,34 @@ var _ = Describe("Template validator", func() {
})
})

PContext("Certificates", func() {
// TODO: Find a simpler way to test the certificate rotation
Context("Certificates", func() {
It("[test_id:4375] Test refreshing of certificates", func() {
By("destroying the CA certificate")
err := coreClient.CoreV1().Secrets(strategy.GetNamespace()).Delete(ctx, validator.SecretName, metav1.DeleteOptions{})
pods, err := GetRunningPodsByLabel(validator.VirtTemplateValidator, validator.KubevirtIo, strategy.GetNamespace())
Expect(err).ToNot(HaveOccurred())
Expect(pods.Items).ToNot(HaveLen(0))

validatorPod := pods.Items[0]
oldCerts, err := getWebhookServerCertificates(&validatorPod)
Expect(err).ToNot(HaveOccurred())

By("deleting the secret with certificate")
secret := &core.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: validator.SecretName,
Namespace: strategy.GetNamespace(),
},
}
err = apiClient.Delete(ctx, secret)
Expect(err).ToNot(HaveOccurred())

By("checking that the secret gets restored with a new certificate")
Eventually(func() string {
sec, err := GetCertFromSecret(validator.SecretName, strategy.GetNamespace())
Expect(err).ToNot(HaveOccurred())
return sec
}, 120*time.Second, 1*time.Second).Should(Not(BeEmpty()))
Eventually(func() (bool, error) {
newCerts, err := getWebhookServerCertificates(&validatorPod)
if err != nil {
return true, err
}
return certsEqual(newCerts, oldCerts), nil
}, 5*time.Minute, 1*time.Second).Should(BeFalse())
})
})
})
Expand Down Expand Up @@ -957,3 +974,32 @@ func TemplateWithoutRules() *templatev1.Template {
validations := `[]`
return addObjectsToTemplates("test-fedora-desktop-small-without-rules", validations)
}

func getWebhookServerCertificates(validatorPod *core.Pod) ([]*x509.Certificate, error) {
conn, err := portForwarder.Connect(validatorPod, validator.ContainerPort)
if err != nil {
return nil, err
}

tlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})
defer tlsConn.Close()

err = tlsConn.Handshake()
if err != nil {
return nil, err
}

return tlsConn.ConnectionState().PeerCertificates, nil
}

func certsEqual(certs1, certs2 []*x509.Certificate) bool {
if len(certs1) != len(certs2) {
return false
}
for i := 0; i < len(certs1); i++ {
if !certs1[i].Equal(certs2[i]) {
return false
}
}
return true
}
13 changes: 13 additions & 0 deletions vendor/github.com/docker/spdystream/CONTRIBUTING.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 518d02e

Please sign in to comment.