Skip to content

Commit

Permalink
feat: Engine K8S ingress for REST API reverse proxy routing (#1970)
Browse files Browse the repository at this point in the history 
## Description:
Create engine K8S ingress for the REST API port so we can interact with
the API via Traefik. The `Host` header is set to `engine`.

Tested with k3d locally:
```
curl -i http://localhost:9730/engine/info -H "Host: engine"
HTTP/1.1 200 OK
...
{"engine_version":"ae7579"}
```

## Is this change user facing?
NO

## References (if applicable):
#1941
  • Loading branch information
@laurentluce
laurentluce authored Jan 2, 2024
1 parent 1440548 commit 4287f88
Show file tree
Hide file tree
Showing 11 changed files with 218 additions and 41 deletions.
7 changes: 3 additions & 4 deletions ...ne-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ const (
//TODO: pass this parameter
enclaveManagerUIPort = 9711
enclaveManagerAPIPort = 8081
restAPIPort = 9779 //TODO: pass this parameter
maxWaitForEngineAvailabilityRetries = 10
timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second
logsStorageDirpath = "/var/log/kurtosis/"
Expand Down Expand Up @@ -159,12 +158,12 @@ func CreateEngine(
)
}

restAPIPortSpec, err := port_spec.NewPortSpec(uint16(restAPIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait)
restAPIPortSpec, err := port_spec.NewPortSpec(engine.RESTAPIPortAddr, consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait)
if err != nil {
return nil, stacktrace.Propagate(
err,
"An error occurred creating the REST API server's http port spec object using number '%v' and protocol '%v'",
restAPIPort,
engine.RESTAPIPortAddr,
consts.EngineTransportProtocol.String(),
)
}
Expand Down Expand Up @@ -207,7 +206,7 @@ func CreateEngine(
privateGrpcDockerPort: docker_manager.NewManualPublishingSpec(grpcPortNum),
enclaveManagerUIDockerPort: docker_manager.NewManualPublishingSpec(uint16(enclaveManagerUIPort)),
enclaveManagerAPIDockerPort: docker_manager.NewManualPublishingSpec(uint16(enclaveManagerAPIPort)),
restAPIDockerPort: docker_manager.NewManualPublishingSpec(uint16(restAPIPort)),
restAPIDockerPort: docker_manager.NewManualPublishingSpec(engine.RESTAPIPortAddr),
}

bindMounts := map[string]string{
Expand Down
18 changes: 12 additions & 6 deletions ...iner-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts/consts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ package consts

import (
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
)

const (
Expand All @@ -14,11 +15,16 @@ const (
// be stored in the port spec label
KurtosisInternalContainerGrpcPortSpecId = "grpc"

// The ID of the GRPC proxy port for Kurtosis-internal containers. This is necessary because
// Typescript's grpc-web cannot communicate directly with GRPC ports, so Kurtosis-internal containers
// need a proxy that will translate grpc-web requests before they hit the main GRPC server
KurtosisInternalContainerGrpcProxyPortSpecId = "grpc-proxy"
HttpApplicationProtocol = "http"
// The ID of the REST API port
KurtosisInternalContainerRESTAPIPortSpecId = "rest-api"

HttpApplicationProtocol = "http"

IngressRulePathAllPaths = "/"
)

var (
IngressRulePathTypePrefix = netv1.PathTypePrefix
)

// This maps a Kubernetes pod's phase to a binary "is the pod considered running?" determiner
Expand Down
122 changes: 117 additions & 5 deletions ...ib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package engine_functions
import (
"context"
"fmt"
"time"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
Expand All @@ -16,8 +18,8 @@ import (
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
apiv1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
"time"
)

const (
Expand All @@ -26,6 +28,8 @@ const (
maxWaitForEngineContainerAvailabilityRetries = 30
timeBetweenWaitForEngineContainerAvailabilityRetries = 1 * time.Second
httpApplicationProtocol = "http"

restAPIPortHost = "engine"
)

var noWait *port_spec.Wait = nil
Expand Down Expand Up @@ -65,6 +69,15 @@ func CreateEngine(
consts.KurtosisServersTransportProtocol.String(),
)
}
privateRESTAPIPortSpec, err := port_spec.NewPortSpec(engine.RESTAPIPortAddr, consts.KurtosisServersTransportProtocol, httpApplicationProtocol, noWait)
if err != nil {
return nil, stacktrace.Propagate(
err,
"An error occurred creating the engine's private rest api port spec object using number '%v' and protocol '%v'",
engine.RESTAPIPortAddr,
consts.KurtosisServersTransportProtocol.String(),
)
}
privatePortSpecs := map[string]*port_spec.PortSpec{
consts.KurtosisInternalContainerGrpcPortSpecId: privateGrpcPortSpec,
}
Expand Down Expand Up @@ -149,6 +162,7 @@ func CreateEngine(
namespaceName,
engineAttributesProvider,
privateGrpcPortSpec,
privateRESTAPIPortSpec,
enginePodLabels,
kubernetesManager,
)
Expand All @@ -165,13 +179,34 @@ func CreateEngine(
}
}()

engineIngress, err := createEngineIngress(
ctx,
namespaceName,
engineAttributesProvider,
privateRESTAPIPortSpec,
kubernetesManager,
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the engine ingress")
}
var shouldRemoveIngress = true
defer func() {
if shouldRemoveIngress {
if err := kubernetesManager.RemoveIngress(ctx, engineIngress); err != nil {
logrus.Errorf("Creating the engine didn't complete successfully, so we tried to delete Kubernetes ingress '%v' that we created but an error was thrown:\n%v", engineIngress.Name, err)
logrus.Errorf("ACTION REQUIRED: You'll need to manually remove Kubernetes ingress with name '%v'!!!!!!!", engineIngress.Name)
}
}
}()

engineResources := &engineKubernetesResources{
clusterRole: clusterRole,
clusterRoleBinding: clusterRoleBindings,
namespace: namespace,
serviceAccount: serviceAccount,
service: engineService,
pod: enginePod,
ingress: engineIngress,
}
engineObjsById, err := getEngineObjectsFromKubernetesResources(map[engine.EngineGUID]*engineKubernetesResources{
engineGuid: engineResources,
Expand Down Expand Up @@ -216,6 +251,7 @@ func CreateEngine(
shouldRemoveClusterRoleBinding = false
shouldRemovePod = false
shouldRemoveService = false
shouldRemoveIngress = false
return resultEngine, nil
}

Expand Down Expand Up @@ -448,18 +484,21 @@ func createEngineService(
namespace string,
engineAttributesProvider object_attributes_provider.KubernetesEngineObjectAttributesProvider,
privateGrpcPortSpec *port_spec.PortSpec,
privateRESTAPIPortSpec *port_spec.PortSpec,
podMatchLabels map[*kubernetes_label_key.KubernetesLabelKey]*kubernetes_label_value.KubernetesLabelValue,
kubernetesManager *kubernetes_manager.KubernetesManager,
) (*apiv1.Service, error) {
engineServiceAttributes, err := engineAttributesProvider.ForEngineService(
consts.KurtosisInternalContainerGrpcPortSpecId,
privateGrpcPortSpec,
consts.KurtosisInternalContainerGrpcProxyPortSpecId, nil)
consts.KurtosisInternalContainerRESTAPIPortSpecId,
privateRESTAPIPortSpec)
if err != nil {
return nil, stacktrace.Propagate(
err,
"An error occurred getting the engine service attributes using private grpc port spec '%+v'",
"An error occurred getting the engine service attributes using private grpc port spec '%+v' and private REST API port spec '%+v'",
privateGrpcPortSpec,
privateRESTAPIPortSpec,
)
}
engineServiceName := engineServiceAttributes.GetName().GetString()
Expand All @@ -468,7 +507,8 @@ func createEngineService(

// Define service ports. These hook up to ports on the containers running in the engine pod
servicePorts, err := shared_helpers.GetKubernetesServicePortsFromPrivatePortSpecs(map[string]*port_spec.PortSpec{
consts.KurtosisInternalContainerGrpcPortSpecId: privateGrpcPortSpec,
consts.KurtosisInternalContainerGrpcPortSpecId: privateGrpcPortSpec,
consts.KurtosisInternalContainerRESTAPIPortSpecId: privateRESTAPIPortSpec,
})
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the engine service's ports using the engine private port specs")
Expand All @@ -490,11 +530,83 @@ func createEngineService(
if err != nil {
return nil, stacktrace.Propagate(
err,
"An error occurred while creating the service with name '%s' in namespace '%s' with ports '%v'",
"An error occurred while creating the service with name '%s' in namespace '%s' with ports '%v' and '%v'",
engineServiceName,
namespace,
privateGrpcPortSpec.GetNumber(),
privateRESTAPIPortSpec.GetNumber(),
)
}
return service, nil
}

func createEngineIngress(
ctx context.Context,
namespace string,
engineAttributesProvider object_attributes_provider.KubernetesEngineObjectAttributesProvider,
privateRESTAPIPortSpec *port_spec.PortSpec,
kubernetesManager *kubernetes_manager.KubernetesManager,
) (*netv1.Ingress, error) {
engineIngressAttributes, err := engineAttributesProvider.ForEngineIngress()
if err != nil {
return nil, stacktrace.Propagate(
err,
"An error occurred getting the engine ingress attributes",
)
}
engineIngressName := engineIngressAttributes.GetName().GetString()
engineIngressLabels := shared_helpers.GetStringMapFromLabelMap(engineIngressAttributes.GetLabels())
engineIngressAnnotations := shared_helpers.GetStringMapFromAnnotationMap(engineIngressAttributes.GetAnnotations())

engineIngressRules, err := getEngineIngressRules(engineIngressName, privateRESTAPIPortSpec)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the user service ingress rules for ingress service with name '%v'", engineIngressName)
}

createdIngress, err := kubernetesManager.CreateIngress(
ctx,
namespace,
engineIngressName,
engineIngressLabels,
engineIngressAnnotations,
engineIngressRules,
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while creating the ingress with name '%s' in namespace '%s'", engineIngressName, namespace)
}

return createdIngress, nil
}

func getEngineIngressRules(
engineIngressName string,
privateRESTAPIPortSpec *port_spec.PortSpec,
) ([]netv1.IngressRule, error) {
var ingressRules []netv1.IngressRule
ingressRule := netv1.IngressRule{
Host: restAPIPortHost,
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: consts.IngressRulePathAllPaths,
PathType: &consts.IngressRulePathTypePrefix,
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: engineIngressName,
Port: netv1.ServiceBackendPort{
Name: "",
Number: int32(privateRESTAPIPortSpec.GetNumber()),
},
},
Resource: nil,
},
},
},
},
},
}
ingressRules = append(ingressRules, ingressRule)

return ingressRules, nil
}
4 changes: 4 additions & 0 deletions ...ls/kubernetes/kubernetes_kurtosis_backend/engine_functions/engine_kubernetes_resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package engine_functions

import (
apiv1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
)

Expand All @@ -21,4 +22,7 @@ type engineKubernetesResources struct {

// Should always be nil if namespace is nil
pod *apiv1.Pod

// Should always be nil if namespace is nil
ingress *netv1.Ingress
}
35 changes: 34 additions & 1 deletion ...b/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/shared_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package engine_functions

import (
"context"
"net"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors"
Expand All @@ -11,7 +13,7 @@ import (
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
"github.com/kurtosis-tech/stacktrace"
apiv1 "k8s.io/api/core/v1"
"net"
netv1 "k8s.io/api/networking/v1"
)

func getEngineObjectsFromKubernetesResources(allResources map[engine.EngineGUID]*engineKubernetesResources) (map[engine.EngineGUID]*engine.Engine, error) {
Expand Down Expand Up @@ -128,6 +130,7 @@ func getMatchingEngineKubernetesResources(
serviceAccount: nil,
service: nil,
pod: nil,
ingress: nil,
}
}
engineResources.namespace = namespacesForId[0]
Expand Down Expand Up @@ -163,6 +166,7 @@ func getMatchingEngineKubernetesResources(
serviceAccount: nil,
service: nil,
pod: nil,
ingress: nil,
}
}
engineResources.clusterRole = clusterRolesForId[0]
Expand Down Expand Up @@ -198,6 +202,7 @@ func getMatchingEngineKubernetesResources(
serviceAccount: nil,
service: nil,
pod: nil,
ingress: nil,
}
}
engineResources.clusterRoleBinding = clusterRoleBindingsForId[0]
Expand Down Expand Up @@ -297,9 +302,37 @@ func getMatchingEngineKubernetesResources(
pod = podsForId[0]
}

// Ingress
ingresses, err := kubernetes_resource_collectors.CollectMatchingIngresses(
ctx,
kubernetesManager,
namespaceName,
engineMatchLabels,
kubernetes_label_key.IDKubernetesLabelKey.GetString(),
map[string]bool{
engineGuidStr: true,
},
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting ingresses matching engine GUID '%v' in namespace '%v'", engineGuid, namespaceName)
}
var ingress *netv1.Ingress
if ingressesForId, found := ingresses[engineGuidStr]; found {
if len(ingressesForId) > 1 {
return nil, stacktrace.NewError(
"Expected at most one engine ingress in namespace '%v' for engine with GUID '%v' but found '%v'",
namespaceName,
engineGuid,
len(ingresses),
)
}
ingress = ingressesForId[0]
}

engineResources.service = service
engineResources.pod = pod
engineResources.serviceAccount = serviceAccount
engineResources.ingress = ingress
}

return result, nil
Expand Down
4 changes: 1 addition & 3 deletions ...rnetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer(
// Get Service Attributes
apiContainerServiceAttributes, err := apiContainerAttributesProvider.ForApiContainerService(
consts.KurtosisInternalContainerGrpcPortSpecId,
privateGrpcPortSpec,
consts.KurtosisInternalContainerGrpcProxyPortSpecId,
nil)
privateGrpcPortSpec)
if err != nil {
return nil, stacktrace.Propagate(
err,
Expand Down
Loading

0 comments on commit 4287f88

Please sign in to comment.